src/config/lxrgmr-code/token/code.c

   1
   2 /*
   3  * TBD:
   4  * @ use xxx_string() serial of functions, instead of buff opr. put caching buff
   5  *   pointer in state structure.
   6  * @ lookup first token in comment string for MACRO.
   7  *  + define MACRO process (#define, #if, #include, #pragma)
   8  * @ state-doc: c89-cmnt/ext-cmnt/dquoted-str/here-doc/help-doc/macro-code
   9  * @ put global env into a struct.
  10  *
  11  * @ xxx
  12  * # $'', $ flag, and using.
  13  * @ {}
  14  * # ${}, $ flag, envar state.
  15  * # foo () {}, func-flag, {} paired-token,
  16  * # {}, iTokenIdnum,
  17  *
  18  * XXX:
  19  * @ readuntil-doc: c99-cmnt/sh-cmnt/backquoted-str/quoted-str/func-code
  20  */
  21
  22 /*
  23  * bracket & quote & string.
  24  * @ //, #, comment prefix (readuntil).
  25  * # use caching buffer.
  26  * # MACRO state
  27  * @ '', ``, $'', $``, get string/code from pair-char to pair-char.
  28  * @ $, state & flag
  29  * @ "", $"", get string in string-state.
  30  * @ (), $(), <(), >(), foo(){} code-block.
  31  * @ $[], $(()), expr string.
  32  * @ $XXX, ${}, parsing env-var format in envar-state.
  33  */
  34
  35
  36 /* TBD: use CACHING_STR */
  37 char str_buff[8192];
  38 int str_idx = 0;
  39
  40 int blank_flag = 0;
  41 char blank_str[8192];
  42
  43 int recent_token = 0;
  44 int token_len = 0;
  45 char *token_str = NULL;
  46 TOKEN_ID_PROC pfn_token_proc = NULL;
  47
  48 #define BYTE_MASK(n)                            ((1<<((n)<<3)) - 1)
  49
  50
  51
  52
  53
  54 char mask_char = '\\';
  55 char pfx_char = 0;
  56 char pair_begin = 0;
  57 char pair_end = 0;
  58 char endstr[MAX_RSVWD_LEN];
  59 int endstr_len = 0;
  60
  61 extern void token_ungetc (char c, SRC_FILE_DESC *pfile);
  62
  63 char read_getc (SRC_FILE_DESC *pfile)
  64 {
  65         extern char token_getc (SRC_FILE_DESC *pfile);
  66
  67         register char c;
  68
  69         if ((pfile->iTokenStrLen - pfile->iPrevIdx) > 0)
  70         {
  71                 c = pfile->aTokenStrBuff[pfile->iPrevIdx];
  72                 pfile->iPrevIdx++;
  73         }
  74         else
  75                 c = token_getc(pfile);
  76
  77         return c;
  78 }
  79
  80 void read_ungetc (char c, SRC_FILE_DESC *pfile)
  81 {
  82         if (pfile->iPrevIdx > 0)
  83         {
  84                 pfile->iPrevIdx--;
  85                 pfile->aTokenStrBuff[pfile->iPrevIdx] = c;
  86         }
  87         else
  88                 token_ungetc(c, pfile);
  89 }
  90
  91 int read_token_str (SRC_FILE_DESC *pfile)
  92 {
  93         return 0;
  94 }
  95
  96 /*
  97  * \retval: 0, in normal.
  98  *                      -1, buffer is full.
  99  */
 100 int readuntil (SRC_FILE_DESC *pfile)
 101 {
 102         int count = 0;
 103         char c;
 104
 105         while (str_idx < sizeof(str_buff)-1)
 106         {
 107                 c = read_getc(pfile);
 108 //              printf("c=%c(%x)(%x)\n", c, c, endstr[0]);
 109                 // ignore NULL char in txt src file.
 110                 if (c == 0)
 111                         continue;
 112                 else if (c == mask_char)
 113                 {
 114                         c = read_getc(pfile);
 115                 }
 116                 else if (c == pfx_char)
 117                 {
 118                         //
 119                 }
 120                 else if (c == pair_begin)
 121                 {
 122                         count++;
 123                 }
 124                 else if (c == pair_end)
 125                 {
 126                         count--;
 127                 }
 128                 /*
 129                  * '',
 130                  * '\n', str='X'
 131                  * '\n', str='\n'
 132                  * 'EOF', str='aaa\n'
 133                  * 'EOF', str='EXX'
 134                  * 'EOF', str='EOL'
 135                  * 'EOF', str='EEOL'
 136                  * 'EOF', str='EOEOL'
 137                  */
 138                 else if (endstr[0] == 0)
 139                 {
 140                         read_ungetc(c, pfile);
 141                         break;
 142                 }
 143                 else if (c == endstr[0])
 144                 {
 145                         int i = 1;
 146
 147                         while (i < endstr_len && (c=read_getc(pfile)) == endstr[i]) i++;
 148
 149                         if (i < endstr_len)
 150                         {
 151                                 read_ungetc(c, pfile);
 152                                 while (--i) read_ungetc(endstr[i], pfile);
 153                                 c = endstr[0];
 154 //                              memcpy(&str_buff[str_idx], endstr, i);
 155 //                              str_idx += i;
 156 //                              str_buff[str_idx] = 0;
 157                         }
 158                         else
 159                                 break;
 160                 }
 161
 162                 str_buff[str_idx] = c;
 163                 str_idx++;
 164 //              str_buff[str_idx] = 0;
 165         }
 166         str_buff[str_idx] = 0;
 167         if (str_idx == sizeof(str_buff)-1)
 168                 return -1;
 169         else
 170                 return 0;
 171 }
 172
 173 /*
 174  * c99 comment string proc.
 175  */
 176 int on_c99_cmnt_proc (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 177 {
 178         printf("################################################ c99 comment\n");
 179
 180         mask_char = 0;
 181         pfx_char = 0;
 182         pair_begin = 0;
 183         pair_end = 0;
 184         endstr[0] = '\n';
 185         endstr[1] = 0;
 186         endstr_len = 1;
 187
 188         /* TBD: use xxx_string() function to store txt. */
 189         /* ignore overlimited comment string. */
 190         while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 191 //      printf("str_idx = %d\n", str_idx);
 192 //      if (str_idx != 0)
 193 //              printf(str_buff);
 194
 195         return T_C99_CMNT;
 196 }
 197
 198 /*
 199  * sh comment string proc.
 200  */
 201 int on_sh_cmnt_proc (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 202 {
 203         int ret;
 204         char c;
 205
 206         int Macro_TokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState);
 207
 208         printf("################################################ sh comment\n");
 209
 210         str_idx = 1;
 211         str_buff[0] = '#';
 212         str_buff[1] = 0;
 213         while (1)
 214         {
 215                 c = read_getc(psrcfile);
 216                 if (c == ' ' || c == '\t')
 217                         break;
 218                 if (c == '\n')
 219                         return T_SH_CMNT;
 220
 221                 str_buff[str_idx] = c;
 222                 str_idx++;
 223         }
 224         str_buff[str_idx] = 0;
 225
 226         ret = Macro_TokenProc(str_buff, str_idx, gpcurr_src_file->pstCurrTokenState);
 227
 228         /*
 229          * no preproc cmd matching, it's a sh-cmnt.
 230          * otherwize, return token.
 231          */
 232         if (ret != 0)
 233                 return ret;
 234
 235         mask_char = 0;
 236         pfx_char = 0;
 237         pair_begin = 0;
 238         pair_end = 0;
 239         endstr[0] = '\n';
 240         endstr[1] = 0;
 241         endstr_len = 1;
 242
 243         /* ignore overlimited comment string. */
 244         while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 245
 246         return T_SH_CMNT;
 247 }
 248
 249
 250 char trans_char_tbl[128] = {
 251 /*      0       1       2               3       4               5       6               7       8               9       A       B       C       D       E               F */
 252         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      0,      0,      0,              0,      // 0
 253         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      0,      0,      0,              0,      // 1
 254 /*      ' '     !       "               #       $               %       &               '       (               )       *       +       ,       -       .               / */
 255         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      0,      0,      0,              0,      // 2
 256 /*      0       1       2               3       4               5       6               7       8               9       :       ;       <       =       >               ? */
 257         -16,-17,-18,    -19,-20,        -21,-22,        -23,-24,        -25,0,  0,      0,      0,      0,              0,      // 3
 258 /*      @       A       B               C       D               E       F               G       H               I       J       K       L       M       N               O */
 259         0,      -26,-27,        -28,-29,        -30,-31,        0,      0,              0,      0,      0,      0,      0,      0,              0,      // 4
 260 /*      P       Q       R               S       T               U       V               W       X               Y       Z       [       \       ]       ^               _ */
 261         0,      0,      0,              0,      0,              0,      0,              0,      0,              0,      0,      0,      1,      0,      0,              0,      // 5
 262 /*      `       a       b               c       d               e       f               g       h               i       j       k       l       m       n               o */
 263         0,      -26,-27,        -28,-29,        -30,'\f',       0,      0,              0,      0,      0,      0,      0,      '\n',   0,      // 6
 264 /*      p       q       r               s       t               u       v               w       x               y       z       {       |       }       ~                 */
 265         0,      0,      '\r',   0,      '\t',   0,      '\v',   0,      -127,   0,      0,      0,      0,      0,      0,              0,      // 7
 266 };
 267
 268 int str2hex (char *str, int iidx, int oidx)
 269 {
 270         int i;
 271         unsigned long long val = 0;
 272         int chars = 0;
 273         char *pbuff = &str[iidx];
 274         static char tmp[32];
 275         register char c;
 276
 277         /*
 278          * str2int
 279          * max hex string len is 32, it's a 128bit hex value string.
 280          */
 281         for (i=0; i<32; i++)
 282         {
 283                 c = trans_char_tbl[pbuff[i]>>1];
 284                 if (c < 0 || c == '\f')
 285                 {
 286                         tmp[i] = ((c<-15) ? (c+16) : 0x0f);
 287                 }
 288                 else
 289                         break;
 290         }
 291
 292         /* calc */
 293         chars = i;
 294         for (; i>=0; i--) val += tmp[i]<<(chars - i);
 295
 296         /* store value */
 297         pbuff = &str[oidx];
 298         for (i=0; i<chars>>2; i++)
 299         {
 300                 pbuff[i] = (char)(val>>i);
 301         }
 302
 303         return chars;
 304 }
 305
 306 int translate_char (char *str, int len)
 307 {
 308         int i, j;
 309         char c;
 310
 311         /*
 312          * "a"
 313          * "\a"
 314          * "\n"
 315          * "\xaa"
 316          */
 317         for (i=0; i<len; i++)
 318         {
 319                 c = str[i];
 320                 if (c == '\\')
 321                 {
 322                         i++;
 323                         {
 324                                 c = trans_char_tbl[c&0x7f];
 325                                 str[j] = (c > 0) ? c : str[i];
 326                                 if (c == -127)
 327                                 {
 328                                         i++;
 329                                         c = str2hex(str, i, j);
 330                                         j += (int)((c+1)>>1);
 331                                         i += (int)c;
 332                                         continue;
 333                                 }
 334                                 c = str[j];
 335                         }
 336                 }
 337                 str[j] = c;
 338                 j++;
 339         }
 340         str[j] = 0;
 341         return j;
 342 }
 343
 344
 345 /*
 346  * quote string proc.
 347  */
 348 int on_quote (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 349 {
 350         printf("################################################ single quote\n");
 351
 352         mask_char = 0;
 353         pfx_char = 0;
 354         pair_begin = 0;
 355         pair_end = 0;
 356         endstr[0] = '\'';
 357         endstr[1] = 0;
 358         endstr_len = 1;
 359
 360         /* ignore overlimited comment string. */
 361         while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 362
 363         /* TBD: with $ flag, translate \X to corresponding char. */
 364         if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 365         {
 366                 //CLR_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR);
 367                 str_idx = translate_char(str_buff, str_idx);
 368                 STATE_RECOVERY(psrcfile);
 369         }
 370
 371         return T_WORD_QUOTE;
 372 }
 373
 374 /*
 375  * quote string proc.
 376  */
 377 int on_bquote (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 378 {
 379         printf("################################################ back quote\n");
 380         mask_char = '\\';
 381         pfx_char = 0;
 382         pair_begin = '"';
 383         pair_end = '"';
 384         endstr[0] = '`';
 385         endstr[1] = 0;
 386         endstr_len = 1;
 387
 388         /* ignore overlimited comment string. */
 389         while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 390
 391         return T_WORD_BACK_QUOTE;
 392 }
 393
 394 int on_ext_cmnt (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 395 {
 396         printf("################################################ ext comment\n");
 397
 398         if (evt == EVT_ON_ENTER)
 399         {
 400                 //
 401                 return T_SKIP;
 402         }
 403         else if (evt == EVT_ON_EXIT)
 404         {
 405                 // do cmnt proc and return token.
 406                 return T_EXT_CMNT;
 407         }
 408 }
 409
 410 int on_c89_cmnt (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 411 {
 412         printf("a ################################################ c89 comment\n");
 413
 414         if (evt == EVT_ON_ENTER)
 415         {
 416                 printf("enter\n");
 417                 strcpy(str_buff, "/*");
 418                 str_idx += 2;
 419                 return T_SKIP;
 420         }
 421         else if (evt == EVT_ON_EXIT)
 422         {
 423                 printf("exit\n");
 424                 // do cmnt proc and return token.
 425                 strcat(str_buff, "*/");
 426                 str_idx += 2;
 427                 return T_C89_CMNT;
 428         }
 429
 430         ////////////////////////////////////
 431         mask_char = 0;
 432         pfx_char = 0;
 433         pair_begin = 0;
 434         pair_end = 0;
 435         endstr[0] = '*';
 436         endstr[1] = '/';
 437         endstr[2] = 0;
 438         endstr_len = 2;
 439
 440         str_buff[0] = '/';
 441         str_buff[1] = '*';
 442         str_idx = 2;
 443         /* TBD: use xxx_string() function to store txt. */
 444         /* ignore overlimited comment string. */
 445         while (readuntil(gpcurr_src_file) == -1) str_idx = 0;
 446         strcat(str_buff, "*/");
 447 //      printf("str_idx = %d\n", str_idx);
 448 //      if (str_idx != 0)
 449 //              printf(str_buff);
 450
 451         return T_C89_CMNT;
 452 }
 453
 454 int on_dquote (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 455 {
 456         printf("################################################ dual quote\n");
 457         return 0;
 458 }
 459
 460
 461
 462
 463
 464 /*
 465  * ():
 466  * code-block, sub-script. stdout is main-script's stdout.
 467  * $():
 468  * code-block, sub-script. stdout is cmd-line buffer.
 469  */
 470 int on_cb_bracket1 (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 471 {
 472         printf("################################################ bracket1 ()\n");
 473
 474         /* subscript */
 475
 476         /* func param list */
 477
 478         return 0;
 479 }
 480
 481 int on_bracket2 (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 482 {
 483         printf("################################################ bracket2 (())\n");
 484
 485         if (evt == EVT_ON_ENTER)
 486         {
 487                 if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 488                 {
 489                         STATE_RECOVERY(psrcfile);
 490                         STATE_SWITCH(psrcfile, STATE_EXPR);
 491                         return 0;
 492                 }
 493         }
 494         else if (evt == EVT_ON_EXIT)
 495         {
 496                 if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 497                 {
 498                         STATE_RECOVERY(psrcfile);
 499                         return 0;
 500                 }
 501         }
 502
 503         return 0;
 504 }
 505
 506 int on_dbracket1 (int evt, int token, register const char *str, register unsigned int len, SRC_FILE_DESC *psrcfile)
 507 {
 508         printf("################################################ dbracket1 [[]]\n");
 509
 510         if (evt == EVT_ON_ENTER)
 511         {
 512                 if (CHK_STATE_FLAG(psrcfile->pstCurrTokenState, TF_DOLLAR))
 513                 {
 514                         STATE_RECOVERY(psrcfile);
 515                         STATE_SWITCH(psrcfile, STATE_EXPR);
 516                         return 0;
 517                 }
 518         }
 519         else if (evt == EVT_ON_EXIT)
 520         {
 521                 if (STATE_CHK(psrcfile, STATE_EXPR))
 522                 {
 523                         STATE_RECOVERY(psrcfile);
 524                         return 0;
 525                 }
 526         }
 527
 528         return 0;
 529 }
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539 char *state_token_str = NULL;
 540 TOKEN_ID_PROC pfn_pair_token_proc = NULL;
 541 int token_state = 0;
 542
 543 /*
 544  * TBD: improve
 545  * @ use three func to process word/pair/other token.
 546  */
 547 int id_str_matching (char *str, int str_len, const struct token_id *pstTokenIdTbl, short *ptoken_len_tbl)
 548 {
 549         int i, k, h;
 550         int len = str_len;
 551         char *pstr;
 552         short *plentbl;
 553         int str_idx_bak = str_idx;
 554
 555         if (!pstTokenIdTbl) return 0;
 556
 557 //      printf("%s: %s(%s)(%d)\n", __FILE__, __FUNCTION__, str, str_len);
 558 //      printf("pstTokenIdTbl = 0x%08x\n", pstTokenIdTbl);
 559 //      printf("ptoken_len_tbl = 0x%08x\n", ptoken_len_tbl);
 560 //      printf("len = 0x%08x\n", len);
 561
 562         for (k=0; len>0; k++)
 563         {
 564                 plentbl = ptoken_len_tbl;
 565 //              printf("str=%s(%d)\n", str, k);
 566                 for (i=0; i<MAX_TOKEN_ID_TBL_SIZE; i++)
 567                 {
 568 //                      printf("XXXXXXXXXXXXXXXXXXXXXXXXXX\n");
 569                         if (pstTokenIdTbl[i].name == NULL &&
 570                                 pstTokenIdTbl[i].tokenstr == NULL &&
 571                                 pstTokenIdTbl[i].token == 0 &&
 572                                 pstTokenIdTbl[i].flags == 0 &&
 573                                 pstTokenIdTbl[i].proc == NULL)
 574                                 break;
 575 //                      printf("pstTokenIdTbl[%d].name = %s(0x%08x)(%s)\n", i, "token", pstTokenIdTbl[i].name, pstTokenIdTbl[i].tokenstr);
 576                         register int j;
 577                         register int token_id_len = *plentbl;
 578                         plentbl++;
 579
 580                         if (len < token_id_len) continue;
 581
 582                         /* TBD: normally, this addr area 'chars[3]' should not be used for user accessing. */
 583                         pstr = (pstTokenIdTbl[i].chars[3]) ? pstTokenIdTbl[i].name : (char *)&pstTokenIdTbl[i].name;
 584 //                      printf("@%d.[%s]: %s(%d)(%d)\n", i, pstTokenIdTbl[i].tokenstr, pstr, token_id_len, pstTokenIdTbl[i].token);
 585
 586                         for (j=0; j<token_id_len; j++)
 587                         {
 588                                 if (pstr[j] != str[j])
 589                                         break;
 590                         }
 591                         if (j == token_id_len)
 592                         {
 593 //                              printf("@%d.[%s]: %s(%d)(%d)[%s]\n", i, pstTokenIdTbl[i].tokenstr, pstr, token_id_len, pstTokenIdTbl[i].token, str);
 594
 595                                 if (pstr[token_id_len] != 0)
 596                                 {
 597                                         state_token_str = &pstr[token_id_len];
 598                                         pfn_pair_token_proc = pstTokenIdTbl[i].proc;
 599                                 }
 600
 601                                 pfn_token_proc = (TOKEN_ID_PROC)pstTokenIdTbl[i].proc;
 602                                 token_len = token_id_len;
 603                                 token_str = pstr;
 604                                 token_state = pstTokenIdTbl[i].state;
 605                                 if (str_idx != 0)
 606                                 {
 607                                         recent_token = pstTokenIdTbl[i].token;
 608                                         return T_WORD;
 609                                 }
 610
 611                                 return pstTokenIdTbl[i].token;
 612                         }
 613                         else if (pstr[token_id_len])
 614                         {
 615                                 for (h=0,j=token_id_len; j<(token_id_len<<1); h++,j++)
 616                                 {
 617                                         if (pstr[j] != str[h])
 618                                                 break;
 619                                 }
 620                                 if (j == (token_id_len<<1))
 621                                 {
 622 //                                      printf("&%d.[%s]: %s(%d)(%d)\n", i, pstTokenIdTbl[i].tokenstr, &pstr[token_id_len], token_id_len, pstTokenIdTbl[i].token+1);
 623
 624                                         pfn_token_proc = (TOKEN_ID_PROC)pstTokenIdTbl[i].proc;
 625                                         token_len = token_id_len;
 626                                         token_str = &pstr[token_id_len];
 627                                         token_state = pstTokenIdTbl[i].state;
 628                                         if (str_idx != 0)
 629                                         {
 630                                                 recent_token = pstTokenIdTbl[i].token+1;
 631                                                 return T_WORD;
 632                                         }
 633
 634                                         return pstTokenIdTbl[i].token+1;
 635                                 }
 636                         }
 637                 }
 638                 str_buff[str_idx] = *str;
 639                 str_idx++;
 640                 if (str_idx >= 8192)
 641                         fprintf(stderr, "err: word string buffer is over limited.\n");
 642                 str++;
 643                 len--;
 644         }
 645
 646         str_idx = str_idx_bak;
 647         return 0;
 648 }
 649
 650 int mask_flag = 0;
 651
 652 /*
 653  * general proc for text/string. it store string into text buffer.
 654  * it can be used in:
 655  * @ comment string.
 656  * @ general string.
 657  * @ here-doc.
 658  */
 659 int Txt_TokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 660 {
 661         if (blank_flag == 1)
 662         {
 663                 strcpy(str_buff, blank_str);
 664                 str_idx += strlen(blank_str);
 665         }
 666         strcat(str_buff, str);
 667         pstTokenState->token_len = strlen(str);
 668         str_idx += pstTokenState->token_len;
 669
 670         printf("Txt_TokenProc(%s) (%s) %d, %d\n", str_buff, str, pstTokenState->token_len, str_idx);
 671         return T_SKIP;
 672 }
 673
 674 int String_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 675 {
 676         return 0;
 677 }
 678
 679
 680 /*
 681  * general proc for text/string. it store string into text buffer.
 682  * it can be used in:
 683  * @ comment string.
 684  * @ general string.
 685  * @ here-doc.
 686  */
 687 int Macro_TokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 688 {
 689         int i;
 690         int ret;
 691         int str_idx_bak;
 692         char MacroCMDBuff[16];
 693
 694 //      printf("Macro_TokenProc(%s)\n", str);
 695
 696         /* matching first following word for marco cmd */
 697         for (i=0; i< str_idx && i<sizeof(MacroCMDBuff); i++)
 698         {
 699                 if (str_buff[i] == ' ') break;
 700                 MacroCMDBuff[i] = str_buff[i];
 701         }
 702         MacroCMDBuff[i] = 0;
 703
 704 //      printf("%s(%d)\n", MacroCMDBuff, i);
 705
 706         str_idx_bak = str_idx;
 707         str_idx = 0;
 708
 709         ret = id_str_matching (
 710                         MacroCMDBuff,
 711                         i,
 712                         macro_array,
 713                         au16MacroTokenLen);
 714
 715         str_idx = str_idx_bak;
 716
 717         return ret;
 718 }
 719
 720 /*
 721  * set mask flag/state, the following char will be treated as a WORD char.
 722  */
 723 int Def_MaskTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 724 {
 725         printf("OnMaskTokenProc(%s)\n", str);
 726
 727         mask_flag = 1;
 728         return T_SKIP;
 729 }
 730
 731 /*
 732  * lookup reserved-word table, maching correspoding token.
 733  * or return as a normal T_WORD.
 734  */
 735 int Def_WordTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 736 {
 737         int len = strlen(str);
 738         int ret = 0;
 739
 740         printf("OnWordTokenProc(%s)\n", str);
 741
 742         /* xxx: macro define replacing. */
 743
 744         pfn_token_proc = NULL;
 745
 746         if (mask_flag == 1)
 747         {
 748                 mask_flag = 0;
 749                 str_buff[str_idx] = *str;
 750                 str_idx++;
 751                 str_buff[str_idx] = 0;
 752                 str++;
 753         }
 754
 755         ret = id_str_matching (
 756                         str,
 757                         len,
 758                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_WORD].pstTokenTbl,
 759                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_WORD]);
 760
 761         pstTokenState->pfnTokenProc = pfn_token_proc;
 762         pstTokenState->recent_token = recent_token;
 763         recent_token = 0;
 764         pstTokenState->token_len = token_len;
 765         token_len = 0;
 766         pstTokenState->token_str = token_str;
 767
 768         if (str_idx != 0)
 769         {
 770                 memcpy(str_buff, str, str_idx);
 771                 str_buff[str_idx] = 0;
 772         }
 773
 774         if (ret == 0)
 775         {
 776                 strcpy(str_buff, str);
 777                 str_idx = strlen(str);
 778                 pstTokenState->token_len = str_idx;
 779                 return T_WORD;
 780         }
 781
 782         return ret;
 783 }
 784
 785 #define UINT_MAX                        (-1 & (~0x80000000))
 786
 787 int state_token = 0;
 788
 789 /*
 790  * there are three kinds of punct string.
 791  * @ prefix-char, this char/str let the program entering a new state.
 792  * @ pair-char, also can trig new state.
 793  * @ operation-char, for grammar element token.
 794  * @ normal-char, T_WORD.
 795  * in this function, it recognize and split string, to get one token by looking
 796  * up pfx-char/pair-char/opr-char table.
 797  */
 798 int Def_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 799 {
 800         int len=strlen(str);
 801         int ret = 0;
 802         int opr, pfx, pair;
 803         int opr_token, pfx_token, pair_token;
 804         int opr_idx, pfx_idx, pair_idx;
 805         int opr_len, pfx_len, pair_len;
 806         TOKEN_ID_PROC opr_proc = NULL, pfx_proc = NULL, pair_proc = NULL;
 807         unsigned int tmp;
 808         char *tmp_token_str[3];
 809
 810         printf("OnPunctTokenProc(%s)\n", str);
 811
 812         pfn_token_proc = NULL;
 813         recent_token = 0;
 814         token_len = 0;
 815
 816         if (mask_flag == 1)
 817         {
 818                 mask_flag = 0;
 819                 str_buff[str_idx] = *str;
 820                 str_idx++;
 821                 str_buff[str_idx] = 0;
 822                 str++;
 823         }
 824
 825         pfx = id_str_matching (
 826                         str,
 827                         len,
 828                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_PFX].pstTokenTbl,
 829                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_PFX]);
 830         pfx_token = recent_token;
 831         pfx_len = token_len;
 832         pfx_idx = str_idx;
 833         recent_token = 0;
 834         str_idx = 0;
 835         pfx_proc = pfn_token_proc;
 836         if (pfx != 0)
 837                 tmp_token_str[0] = token_str;
 838 //      printf("pfx = %d\n", pfx);
 839 //      printf("str_idx = %d\n", str_idx);
 840 //      printf("pfx_len = %d\n", pfx_len);
 841
 842         /* TBD: backup */
 843         pair = id_str_matching (
 844                         str,
 845                         len,
 846                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_PAIR].pstTokenTbl,
 847                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_PAIR]);
 848         pair_token = recent_token;
 849         pair_len = token_len;
 850         pair_idx = str_idx;
 851         recent_token = 0;
 852         str_idx = 0;
 853         pair_proc = pfn_token_proc;
 854         if (pair != 0)
 855                 tmp_token_str[1] = token_str;
 856 //      printf("pair = %d\n", pair);
 857 //      printf("str_idx = %d\n", str_idx);
 858 //      printf("pair_len = %d\n", pair_len);
 859
 860         opr = id_str_matching (
 861                         str,
 862                         len,
 863                         pstTokenState->pstStateInfo->astTokenTypeTbl[TYPE_PUNCT].pstTokenTbl,
 864                         pstTokenState->pstStateInfo->api16TokenNameLen[TYPE_PUNCT]);
 865         opr_token = recent_token;
 866         opr_len = token_len;
 867         opr_idx = str_idx;
 868         recent_token = 0;
 869         str_idx = 0;
 870         opr_proc = pfn_token_proc;
 871         if (opr != 0)
 872                 tmp_token_str[2] = token_str;
 873 //      printf("opr = %d\n", opr);
 874 //      printf("str_idx = %d\n", str_idx);
 875 //      printf("opr_len = %d\n", opr_len);
 876
 877         tmp = -1;
 878         if (pfx == 0)
 879         {
 880                 pfx_idx = UINT_MAX;
 881                 pfx_len = 0;
 882         }
 883         if (pair == 0)
 884         {
 885                 pair_idx = UINT_MAX;
 886                 pair_len = 0;
 887         }
 888         if (opr == 0)
 889         {
 890                 opr_idx = UINT_MAX;
 891                 opr_len = 0;
 892         }
 893
 894 //      printf("pfx = %d,%d,%d\n", pfx, pfx_idx, pfx_len);
 895 //      printf("pair = %d,%d,%d\n", pair, pair_idx, pair_len);
 896 //      printf("opr = %d,%d,%d\n", opr, opr_idx, opr_len);
 897         if (pfx != 0 &&
 898                 pfx_idx <= pair_idx && pfx_idx <= opr_idx &&
 899                 pfx_len >= pair_len && pfx_len >= opr_len)
 900         {
 901 //              printf("xxxxxxxxxxxxxxxx1(%d)\n", pfx);
 902                 recent_token = pfx_token;
 903                 token_len = pfx_len;
 904                 str_idx = pfx_idx;
 905                 pfn_token_proc = pfx_proc;
 906                 pstTokenState->token_str = tmp_token_str[0];
 907                 ret = pfx;
 908         }
 909         if (pair != 0 &&
 910                 pair_idx <= pfx_idx && pair_idx <= opr_idx &&
 911                 pair_len >= pfx_len && pair_len >= opr_len)
 912         {
 913                 /* entering paired token, and need to be switched to new state. */
 914                 if (token_state != 0)
 915                 {
 916                         if (state_token_str)
 917                         {
 918                                 pstTokenState->pfnPairTokenProc = pfn_pair_token_proc;
 919                                 pstTokenState->state_token_str = state_token_str;
 920                                 pstTokenState->paired_token = pair+1;
 921                         }
 922
 923                         pstTokenState->paired_state = token_state;
 924                 }
 925 //              printf("xxxxxxxxxxxxxxxx2 (%s) pstTokenState->paired_token (%d)\n", pstTokenState->state_token_str, pstTokenState->paired_token);
 926                 recent_token = pair_token;
 927                 token_len = pair_len;
 928                 str_idx = pair_idx;
 929                 pfn_token_proc = pair_proc;
 930                 pstTokenState->token_str = tmp_token_str[1];
 931                 ret = pair;
 932         }
 933         if (opr != 0 &&
 934                 opr_idx <= pfx_idx && opr_idx <= pair_idx &&
 935                 opr_len >= pfx_len && opr_len >= pair_len)
 936         {
 937 //              printf("xxxxxxxxxxxxxxxx3(%d)\n", opr);
 938                 recent_token = opr_token;
 939                 token_len = opr_len;
 940                 str_idx = opr_idx;
 941                 pfn_token_proc = opr_proc;
 942                 pstTokenState->token_str = tmp_token_str[2];
 943                 ret = opr;
 944         }
 945 //      printf("matching ret = %d\n", ret);
 946
 947         pstTokenState->pfnTokenProc = pfn_token_proc;
 948         pstTokenState->recent_token = recent_token;
 949         recent_token = 0;
 950         pstTokenState->token_len = token_len;
 951         token_len = 0;
 952         state_token_str = 0;
 953         token_state = 0;
 954
 955         if (str_idx != 0)
 956         {
 957                 memcpy(str_buff, str, str_idx);
 958                 str_buff[str_idx] = 0;
 959         }
 960
 961         if (ret == pfx && ret != 0)
 962         {
 963                 state_token = ret;
 964 //              ret = T_SKIP;
 965         }
 966
 967 //      printf("pstTokenState->token_len = %d;\n", pstTokenState->token_len);
 968
 969         if (!ret)
 970         {
 971                 strcpy(str_buff, str);
 972                 str_idx = strlen(str);
 973                 pstTokenState->token_len = str_idx;
 974 //              printf("return T_WORD(%s)\n", str);
 975                 return T_WORD;
 976         }
 977
 978 //      printf("return ret = %d\n", ret);
 979
 980         return ret;
 981 }
 982
 983 int Def_SpaceTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
 984 {
 985         printf("OnSpaceTokenProc(%s)\n", str);
 986
 987         if (mask_flag == 1)
 988         {
 989                 mask_flag = 0;
 990                 str_buff[str_idx] = *str;
 991                 str_idx++;
 992                 str_buff[str_idx] = 0;
 993                 str++;
 994         }
 995
 996         blank_flag = 1;
 997         strcpy(blank_str, str);
 998         str_idx = strlen(str);
 999
1000         pstTokenState->token_len = str_idx;
1001
1002         return T_BLANK;
1003 }
1004
1005 int Def_NewlineTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1006 {
1007         printf("OnNewlineTokenProc(%s)\n", str);
1008
1009         if (mask_flag == 1)
1010         {
1011                 mask_flag = 0;
1012                 str_buff[str_idx] = *str;
1013                 str_idx++;
1014                 str_buff[str_idx] = 0;
1015                 str++;
1016                 if (str[0] == 0)
1017                         return T_SKIP;
1018         }
1019
1020         pstTokenState->token_len = 1;
1021         pstTokenState->nl_cnt = strlen(str) - 1;
1022 //      printf("NL(%d)\n", pstTokenState->nl_cnt);
1023
1024         /* in cmd state, new line reset token id num */
1025         // XXX:
1026         /*
1027         if (STATE_VCHK(pstTokenState, STATE_CMD))
1028                 pstTokenState->iTokenIdNum = 0;
1029         */
1030
1031         return T_EOL;
1032 }
1033
1034 int Def_EOFTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1035 {
1036         printf("OnEOFTokenProc(%s)\n", str);
1037
1038         return 0;
1039 }
1040
1041 /*
1042  *
1043  */
1044 int Envar_WordTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1045 {
1046         int ret = Def_WordTokenProc(str, type, pstTokenState);
1047
1048         /* store to prev state. */
1049         /*
1050         if (STATE_VCHK(pstTokenState, STATE_DOLLAR))
1051         {
1052                 STATE_RECOVERY(gpcurr_src_file);
1053         }
1054         */
1055
1056         /* envar expanssion */
1057         if (0)
1058         {
1059                 /* ungetc() to buffer */
1060                 return T_SKIP;
1061         }
1062         else
1063         {
1064                 return T_WORD;
1065         }
1066 }
1067
1068 /*
1069  *
1070  */
1071 int Envar_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1072 {
1073         int ret = Def_PunctTokenProc(str, type, pstTokenState);
1074
1075         /* store to prev state. */
1076         /*
1077         if (STATE_VCHK(pstTokenState, STATE_DOLLAR))
1078         {
1079                 STATE_RECOVERY(gpcurr_src_file);
1080         }
1081         */
1082
1083         /* $( */
1084         if (ret == T_LBRACKET1)
1085         {
1086                 STATE_SWITCH(gpcurr_src_file, STATE_CODE);
1087                 return T_SKIP;
1088         }
1089         /* $(( or $[ */
1090         else if (ret == T_DLBRACKET1 || ret == T_LBRACKET2)
1091         {
1092                 STATE_SWITCH(gpcurr_src_file, STATE_EXPR);
1093                 return T_SKIP;
1094         }
1095         /* ${ */
1096         else if (ret == T_LBRACKET3)
1097         {
1098                 /* XXX: envar expanssion */
1099                 if (0)
1100                 {
1101                         return T_SKIP;
1102                 }
1103                 else
1104                 {
1105                         return T_WORD;
1106                 }
1107         }
1108
1109         /* XXX: expanssion */
1110         if (0)
1111         {
1112                 return T_SKIP;
1113         }
1114         else
1115         {
1116                 return T_WORD;
1117         }
1118
1119         return ret;
1120 }
1121
1122 int Block_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1123 {
1124         int ret = Def_WordTokenProc(str, type, pstTokenState);
1125
1126         Txt_TokenProc(str, type, pstTokenState);
1127
1128         /* TBD: append string to buffer */
1129
1130         return ret;
1131 }
1132
1133 /*
1134  *
1135  */
1136 int Cmnt_WordTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1137 {
1138         //int ret = Def_WordTokenProc(str, type, pstTokenState);
1139
1140         /* TBD: append string to buffer */
1141
1142         return Txt_TokenProc(str, type, pstTokenState);
1143
1144         //return ret;
1145 }
1146
1147 /*
1148  *
1149  */
1150 int Cmnt_PunctTokenProc (char *str, TOKEN_TYPE type, TOKEN_PROC_STATE *pstTokenState)
1151 {
1152         //int ret = Def_PunctTokenProc(str, type, pstTokenState);
1153
1154         /* TBD: append string to buffer */
1155
1156         return Txt_TokenProc(str, type, pstTokenState);
1157
1158         //return ret;
1159 }
1160
1161 #include "codeblock.c"
1162 #include "envar.c"