src/config/lxrgmr-code/tokenproc/strtoken.c

   1 /************************************************************************
   2  *                         $PROJECT Project
   3  *
   4  *   (c) Copyright 2009, $COMPANY_EN. Inc., $CITY_EN, $COUNTRY_EN
   5  *   (c) Copyright 2009, $COMPANY_CN. Inc., $CITY_CN, $COUNTRY_CN
   6  ************************************************************************
   7  * filename:    symbol.c
   8  * function:    Symbol模块。
   9  * createdate:  $DATE
  10  * author:              $AUTHOR
  11  * note：this file is designed for env variable expanssion.
  12  *
  13  *
  14  ************************************************************************/
  15 /*                                                              Modify record                                                   */
  16 /************************************************************************
  17  * date:                        $DATE
  18  * author:              $AUTHOR
  19  * note:
  20  *
  21  ************************************************************************/
  22
  23 /*
  24 模块引入函数：
  25
  26  */
  27
  28 /*
  29 数据接口：
  30
  31  */
  32
  33 /*******************************
  34  *******************************
  35  * 1. Include Files
  36  *******************************
  37  *******************************/
  38 //#define MODULE_NAME                   MODULE_CLASS_XXX
  39
  40 /* System include files */
  41 #include "type.h"
  42 #include <stdio.h>
  43 #include <string.h>
  44 #include <memory.h>
  45 #include <errno.h>
  46 #include <stdlib.h>
  47 #include <unistd.h>
  48
  49 /* Project module include files */
  50 #include "strtoken.h"
  51
  52 /* Private include files */
  53 //#include <XXX_Private.h>
  54
  55
  56 /*******************************
  57  *******************************
  58  * 2. Global Variables &
  59  *     Micro Defines &
  60  *     Type Defines &
  61  *     Type Proto
  62  *******************************
  63  *******************************/
  64
  65 char map_tbl[128] = {
  66 /*      0       1       2       3       4       5       6       7       8       9       A       B       C       D       E       F */
  67         0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      5,      4,      0,      5,      0,      0,      // 0
  68         0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      0,      // 1
  69 /*      ' '     !       "       #       $       %       &       '       (       )       *       +       ,       -       .       / */
  70         4,      3,      3,      3,      3,      3,      3,      3,      3,      3,      3,      3,      3,      3,      3,      3,      // 2
  71 /*      0       1       2       3       4       5       6       7       8       9       :       ;       <       =       >       ? */
  72         2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      3,      3,      3,      3,      3,      3,      // 3
  73 /*      @       A       B       C       D       E       F       G       H       I       J       K       L       M       N       O */
  74         3,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      // 4
  75 /*      P       Q       R       S       T       U       V       W       X       Y       Z       [       \       ]       ^       _ */
  76         2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      3,      1,      3,      3,      2,      // 5
  77 /*      `       a       b       c       d       e       f       g       h       i       j       k       l       m       n       o */
  78         3,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      // 6
  79 /*      p       q       r       s       t       u       v       w       x       y       z       {       |       }       ~ */
  80         2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      2,      3,      3,      3,      3,      0,      // 7
  81 };
  82
  83 struct token_id *pstTokenNameTbl;
  84
  85 //int iCurrState = 0;
  86 TOKEN_PROC_STATE *gpstTokenStateTbl;
  87
  88 char *gpszTokenStr = NULL;
  89
  90 char buff[8192];
  91
  92 SRC_FILE_DESC *gpcurr_src_file = NULL;
  93
  94 enum {
  95         EVT_ON_ENTER,
  96         EVT_ON_EXIT,
  97 };
  98
  99 #include "AQStack.h"
 100
 101 #include "token/token.var"
 102
 103 //#include "token/quoted-str.var"
 104 //#include "token/dollar.var"
 105 //#include "token/cmd.var"
 106
 107
 108 /*******************************
 109  *******************************
 110  * 3. Private Functions Implement
 111  *******************************
 112  *******************************/
 113
 114
 115
 116 /*******************************
 117  *******************************
 118  * 4. Public Functions Implement
 119  *******************************
 120  *******************************/
 121
 122
 123 #define INIT_PRINTF             //printf
 124
 125
 126
 127 #include "token/code.c"
 128
 129 /*
 130 #include "token/cmnt.c"
 131 #include "token/quoted-str.c"
 132 #include "token/dollar.c"
 133 #include "token/cmd.c"
 134 */
 135
 136 void token_ungetstr (char *str, int len, SRC_FILE_DESC *pfile)
 137 {
 138         int i;
 139         register char c;
 140
 141         for (i=0; i<len; i++)
 142         {
 143                 c = *str;
 144                 str++;
 145                 if (c == '\n')
 146                         pfile->lineno--;
 147
 148                 ungetc(c, pfile->file);
 149         }
 150 }
 151
 152 void token_ungetc (char c, SRC_FILE_DESC *pfile)
 153 {
 154         if (c == '\n')
 155                 pfile->lineno--;
 156
 157         ungetc(c, pfile->file);
 158 }
 159
 160 char token_getc (SRC_FILE_DESC *pfile)
 161 {
 162         register char c=getc(pfile->file);
 163
 164         if (c == '\n')
 165                 pfile->lineno++;
 166
 167         return c;
 168         // chk if need a new blk.
 169         // async read a new blk.
 170         // get curr char.
 171 }
 172
 173 void get_token_word (SRC_FILE_DESC *pfile)
 174 {
 175         char c;
 176         char cc;
 177         static int type = 0;
 178
 179         /*
 180          * if file content include NULL char, it should be ignored.
 181          */
 182 //      if (pfile->bakchar)
 183 //      {
 184 //              c = pfile->bakchar;
 185 //              pfile->bakchar = 0;
 186 //      }
 187 //      else
 188 //              c=token_getc(pfile->file);
 189
 190         pfile->iTokenStrLen = 0;
 191         pfile->iPrevIdx = 0;
 192         pfile->type = 0;
 193
 194         while (c=token_getc(pfile))
 195         {
 196                 pfile->lasttype = pfile->type;
 197                 if (c & 0x80)
 198                 {
 199                         if (c == EOF) // { printf("EOF(%x)\n", c); exit(0); }
 200                         {
 201                                 pfile->type = TYPE_EOF;
 202                                 break;
 203                         }
 204                         else
 205                         {
 206                                 /* XXX: MultiByte */
 207                                 cc=token_getc(pfile);
 208                                 pfile->type = TYPE_WORD;
 209                         }
 210                 }
 211                 else
 212                         pfile->type = map_tbl[c&0x7F];
 213 //              printf("c(%x) = %c(%d, 0x%08x)\n", (unsigned int)c, c, pfile->type, (unsigned int)pfile->lasttype);
 214
 215                 // initial
 216                 if (pfile->lasttype == 0)
 217                 {
 218                         pfile->aTokenStrBuff[0] = c;
 219                         pfile->iTokenStrLen = 1;
 220                         pfile->iPrevIdx = 0;
 221                 }
 222                 else
 223                 {
 224                         if (pfile->type == pfile->lasttype)
 225                         {
 226                                 pfile->aTokenStrBuff[pfile->iTokenStrLen] = c;
 227                                 pfile->iTokenStrLen++;
 228                         }
 229                         else
 230                         {
 231                                 /* ungetc(), the different type char. */
 232                                 token_ungetc(c, pfile);
 233                                 pfile->type = pfile->lasttype;
 234                                 break;
 235                         }
 236                 }
 237         }
 238
 239         pfile->aTokenStrBuff[pfile->iTokenStrLen] = 0;
 240         pfile->iPrevIdx = 0;
 241 //      gpszTokenStr = gpcurr_src_file->aTokenStrBuff;
 242 }
 243
 244 int before_ret_proc (SRC_FILE_DESC *psrcfile, TOKEN_PROC_STATE *pstTokenState, int ret)
 245 {
 246         int tmp = 0;
 247
 248         /*
 249          * switching to new state, then invoke proc func,
 250          * keeps proc func invoked in the same state.
 251          */
 252         if (pstTokenState->paired_state)
 253         {
 254                 STATE_SWITCH(psrcfile, pstTokenState->paired_state);
 255                 printf("switch to strStateName = %s\n", psrcfile->pstCurrTokenState->pstStateInfo->strStateName);
 256                 pstTokenState = psrcfile->pstCurrTokenState;
 257         }
 258
 259         if (pstTokenState->paired_token != 0 && pstTokenState->paired_token == ret)
 260         {
 261                 tmp = pstTokenState->pfnPairTokenProc(
 262                         EVT_ON_EXIT,
 263                         ret,
 264                         &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx],
 265                         psrcfile->iTokenStrLen - psrcfile->iPrevIdx,
 266                         psrcfile);
 267
 268                 /* switching back to latest state */
 269                 STATE_RECOVERY(psrcfile);
 270                 printf("recovery to strStateName = %s\n", psrcfile->pstCurrTokenState->pstStateInfo->strStateName);
 271
 272                 psrcfile->pstCurrTokenState->state_token_str = 0;
 273                 pstTokenState->pfnPairTokenProc = NULL;
 274         }
 275         else
 276         {
 277                 tmp = pstTokenState->pfnTokenProc(
 278                         EVT_ON_ENTER,
 279                         ret,
 280                         &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx],
 281                         psrcfile->iTokenStrLen - psrcfile->iPrevIdx,
 282                         psrcfile);
 283                 pstTokenState->pfnTokenProc = NULL;
 284 //              printf("token = %d;\n", ret);
 285         }
 286         if (tmp != 0)
 287                 ret = tmp;
 288
 289 //      if (str_idx == 0)
 290         {
 291
 292         }
 293
 294         return ret;
 295 }
 296
 297 #define paired_token_matching(str, len, token)  strncmp(str, token, len);
 298
 299 int token_read (SRC_FILE_DESC *psrcfile)
 300 {
 301         int tmp;
 302         int ret = 0;
 303         TOKEN_PROC_STATE *pstTokenState = psrcfile->pstCurrTokenState;
 304
 305 //      printf("%s: %s()\n", __FILE__, __FUNCTION__);
 306
 307         //str_idx = 0;
 308         //str_buff[0]=0;
 309         free_string(&strbuff);
 310         do {
 311                 ret = 0;
 312 #if 0
 313                 /* single step for token debugging. */
 314                 getchar();
 315 #endif
 316
 317 //              printf("iPrevIdx = %d; iTokenStrLen = %d; recent_token = %d; nl_cnt = %d;\n",
 318 //                      psrcfile->iPrevIdx,
 319 //                      psrcfile->iTokenStrLen,
 320 //                      pstTokenState->recent_token,
 321 //                      pstTokenState->nl_cnt);
 322
 323                 /*
 324                  * @ token string buffer getting.
 325                  * # no token to be returned (recent_token==0).
 326                  * # no bufferd string to be analyzed (PrevIdx == TokenStrLen).
 327                  * it trig new buffer appending.
 328                  */
 329                 if (psrcfile->iPrevIdx >= psrcfile->iTokenStrLen &&
 330                         pstTokenState->recent_token == 0 &&
 331                         pstTokenState->nl_cnt == 0)
 332                         get_token_word(psrcfile);
 333
 334 #if 0
 335                 printf("aTokenStrBuff{%d} = \"%s\"\n",
 336                         psrcfile->iTokenStrLen - psrcfile->iPrevIdx,
 337                         &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx]);
 338
 339 #if 0
 340                 //strcpy(str_buff, &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx]);
 341                 //str_idx = strlen(str_buff);
 342                 append_string(  &strbuff,
 343                                                 &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx],
 344                                                 psrcfile->iTokenStrLen - psrcfile->iPrevIdx);
 345
 346                 psrcfile->iTokenStrLen = 0;
 347                 if (psrcfile->type == TYPE_EOF)
 348                         return 0;
 349                 return T_WORD;
 350 #endif
 351 #endif
 352
 353                 /*
 354                  * @ analyzing.
 355                  * if PrevIdx < TokenStrLen, it means token string buffer need to be analyzed.
 356                  * if recent_token is not, it means the latest token have not been returned,
 357                  *    it should not analyze in current time.
 358                  */
 359                 if ((psrcfile->iPrevIdx < psrcfile->iTokenStrLen &&
 360                         pstTokenState->recent_token == 0 &&
 361                         pstTokenState->nl_cnt == 0) ||
 362                         psrcfile->type == TYPE_EOF)
 363                 {
 364                         /*
 365                          * it should be reset to 0 before new analyzing.
 366                          */
 367                         //str_idx = 0;
 368                         //str_buff[0]=0;
 369
 370                         if (psrcfile->pstCurrTokenState->state_token_str)
 371                         {
 372                                 ret = paired_token_matching(
 373                                                 &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx],
 374                                                 strlen(psrcfile->pstCurrTokenState->state_token_str),
 375                                                 psrcfile->pstCurrTokenState->state_token_str
 376                                         );
 377                                 if (ret != 0)
 378                                         ret = 0;
 379                                 else
 380                                 {
 381                                         ret = psrcfile->pstCurrTokenState->paired_token;
 382                                         pstTokenState->token_len = strlen(psrcfile->pstCurrTokenState->state_token_str);
 383                                 }
 384
 385 //                              if (ret != 0)
 386 //                                      printf("paired token(%d, %d): %s\n", ret, pstTokenState->token_len, psrcfile->pstCurrTokenState->state_token_str);
 387                         }
 388
 389                         /* token str proc */
 390                         if (ret == 0)
 391                         {
 392                                 ret = psrcfile->pstStateInfo->astTokenTypeTbl[psrcfile->type].pfnTokenStrProc(
 393                                                 &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx],
 394                                                 psrcfile->type,
 395                                                 psrcfile->pstCurrTokenState);
 396                         }
 397                 }
 398
 399 #if 0
 400                 printf("iPrevIdx = %d; iTokenStrLen = %d; token_len = %d; recent_token = %d; ret = %d\n",
 401                         psrcfile->iPrevIdx,
 402                         psrcfile->iTokenStrLen,
 403                         pstTokenState->token_len,
 404                         pstTokenState->recent_token,
 405                         ret);
 406
 407 #if 0
 408 //              psrcfile->iPrevIdx = 0;
 409 //              psrcfile->iTokenStrLen = 0;
 410                 pstTokenState->recent_token = 0;
 411                 return ret;
 412 #endif
 413 #endif
 414
 415 //              printf("return val = %d, %d; (%s)\n", ret, psrcfile->pstCurrTokenState->token_len, &psrcfile->aTokenStrBuff[psrcfile->iPrevIdx]);
 416
 417                 /*
 418                  * @ resualt token outputting.
 419                  * when analyzing, there are two kinds of punct string:
 420                  * # <PUNCT>...always return ret=token.
 421                  * # <WORD><PUNCT>...always return ret=T_WORD token, and restore PUNCT token
 422                  *   in recent_token.
 423                  * when no analyzing work, return recent_token.
 424                  * when no analyzing work, no recent_token, return 0.
 425                  */
 426                 if (psrcfile->type == TYPE_EOF)
 427                 {
 428                         printf("TYPE_EOF\n");
 429                         return 0;
 430                 }
 431                 else if (psrcfile->type == TYPE_SPACE)
 432                 {
 433                         /*
 434                          * for space string, just store it in buffer,
 435                          * it will be accessed when need in next token.
 436                          */
 437                         psrcfile->iPrevIdx += pstTokenState->token_len;
 438                         //str_idx = 0;
 439                         continue;
 440                 }
 441                 /* newline token */
 442                 if (pstTokenState->nl_cnt > 0)
 443                 {
 444                         pstTokenState->nl_cnt--;
 445                         pstTokenState->recent_token = T_EOL;
 446                         pstTokenState->token_len = 1;
 447                         psrcfile->iPrevIdx += pstTokenState->token_len;
 448                         return T_EOL;
 449                 }
 450
 451 #if 0
 452 //              printf("xxxxxxxxxxxxxxxx %d %d\n", psrcfile->iPrevIdx, pstTokenState->token_len);
 453                 ret = (ret == 0) ? pstTokenState->recent_token : ret;
 454                 psrcfile->iPrevIdx += pstTokenState->token_len;
 455                 pstTokenState->recent_token = 0;
 456                 return ret;
 457 #endif
 458
 459                 /*
 460                  * token post-proc.
 461                  */
 462                 if (ret != 0)
 463                 {
 464                         if (ret == T_SKIP)
 465                         {
 466                                 psrcfile->iPrevIdx += pstTokenState->token_len;
 467 //                              printf("a psrcfile->iPrevIdx = %d, %d;\n", psrcfile->iPrevIdx, pstTokenState->token_len);
 468                                 continue;
 469                         }
 470
 471                         /* invoke proc func */
 472                         if ((pstTokenState->recent_token == 0 && pstTokenState->pfnTokenProc != NULL)
 473                                 || pstTokenState->pfnPairTokenProc != NULL)
 474                                 ret = before_ret_proc(psrcfile, pstTokenState, ret);
 475
 476                         tmp = pstTokenState->token_len;
 477                         pstTokenState = psrcfile->pstCurrTokenState;
 478                         pstTokenState->token_len = tmp;
 479
 480                         /*
 481                          * str_idx will be used when T_WORD token return,
 482                          * do not reset it here, and reset it later.
 483                          */
 484 //                      str_idx = 0;
 485                         psrcfile->iPrevIdx += (ret == T_WORD) ? strbuff.text_size : pstTokenState->token_len;
 486 //                      if (ret != T_WORD && ret != T_SKIP)
 487                         {
 488 //                              printf("############################################ str_idx = 0;\n");
 489 //                              str_idx = 0;
 490                         }
 491
 492                         if (ret == T_SKIP)
 493                         {
 494 //                              psrcfile->iPrevIdx += pstTokenState->token_len;
 495 //                              printf("psrcfile->iPrevIdx = %d, %d;\n", psrcfile->iPrevIdx, pstTokenState->token_len);
 496                                 continue;
 497                         }
 498
 499                         /*
 500                          * if token combined out of this function,
 501                          * it should process iTokenIdNum in code.
 502                          */
 503                         psrcfile->pstCurrTokenState->iTokenIdNum++;
 504                         return ret;
 505                 }
 506                 else if (pstTokenState->recent_token != 0)
 507                 {
 508                         ret = pstTokenState->recent_token;
 509                         pstTokenState->recent_token = 0;
 510 //                      psrcfile->iPrevIdx += pstTokenState->token_len;
 511                         //str_idx = 0;
 512                         if (ret == T_SKIP)
 513                         {
 514                                 psrcfile->iPrevIdx += pstTokenState->token_len;
 515                                 pstTokenState->token_len = 0;
 516                                 continue;
 517                         }
 518                         pstTokenState->token_len = 0;
 519
 520                         if (pstTokenState->pfnTokenProc != NULL || pstTokenState->pfnPairTokenProc != NULL)
 521                                 ret = before_ret_proc(psrcfile, pstTokenState, ret);
 522
 523                         tmp = pstTokenState->token_len;
 524                         pstTokenState = psrcfile->pstCurrTokenState;
 525 //                      printf("pstStateInfo->strStateName = %s;\n", pstTokenState->pstStateInfo->strStateName);
 526                         pstTokenState->token_len = tmp;
 527
 528                         psrcfile->iPrevIdx += pstTokenState->token_len;
 529 //                      if (ret != T_WORD && ret != T_SKIP)
 530 //                              str_idx = 0;
 531
 532                         if (ret == T_SKIP)
 533                         {
 534                                 psrcfile->iPrevIdx += pstTokenState->token_len;
 535                                 continue;
 536                         }
 537
 538                         psrcfile->pstCurrTokenState->iTokenIdNum++;
 539                         return ret;
 540                 }
 541                 break;
 542         }
 543         while (1);
 544
 545         printf("return zero.\n");
 546         /* if ret & recent_token are all zero, it should return 0 */
 547         return 0;
 548 }
 549
 550 extern int on_token_proc (int ret);
 551
 552
 553 /*
 554  * TBD: token item sorting.
 555  * "+=" should be putted before "+", that program can matching first for long string word.
 556  * or, use idx-tbl map to a proc tbl, and point to corresponding TokenTbl item at last.
 557  * or, use sorted str-tbl, first chars are created in a idx-tbl.
 558
 559  * @ get token name string tbl, and sorting:
 560 enum
 561 for
 562 foreach
 563 fix
 564 struct
 565 typedef
 566 union
 567  * @ create a first level idx-tbl.
 568 [00] [e, ,06]
 569 [01] [f, ,07]
 570 [02] [s, ,0b]
 571 [03] [t, ,0c]
 572 [04] [u, ,0d]
 573 [05] [ , ,NULL]
 574
 575 [06] [ ,e,13],  ==> 'enum'
 576 [07] [o,f,09]
 577 [08] [i,f,0d]
 578 [09] [r,o,0a]
 579 [0a] [ ,r,14],  ==> 'for'
 580 [0b] [e,r,0c]
 581 [0c] [ ,e,15],  ==> 'foreach'
 582 [0d] [ ,i,16]   ==> 'fix'
 583 [0e] [ ,e,15]xxxxxxxxxxxxxxxx
 584 [0f] [ ,s,17],  ==> 'struct'
 585 [10] [ ,t,18],  ==> 'typedef'
 586 [11] [ ,u,19],  ==> 'unioin'
 587 [12] [ , ,NULL]
 588
 589 [13] [ADDR], point to 'enum' item struct.
 590 [14] [ADDR], 'for'
 591 [15] [ADDR], 'foreach'
 592 [16] [ADDR], 'fix'
 593 [17] [ADDR], 'struct'
 594 [18] [ADDR], 'typedef'
 595 [19] [ADDR], 'union'
 596  * @ [char,lastchar,idx]
 597  *   char, current matching char. if equal to zero, it means matching.
 598  *   lastchar, if it is different in next idx info, that means
 599  *   idx, idx value in tbl for next char. if matching, it point to a idx with token item addr.
 600  * @ algrithm
 601  *   # ergodic every idx item in loop, util lastchar is different.
 602  *   # matching 'char' and given string,
 603  *   # invoke func, and processing matched idx item in embedded by algrithm of XXXXXXX.
 604  * @ f(m,n,l)=min(m,l)*n,
 605  *   min(m,l), string compare times
 606  *   n, item num.
 607  *   eg: m=5,n=20, it will running one handrod time for loop.
 608  * @ by using idx-tbl, it matching first char by list, and so on the rest.
 609  *   f(m,n,l)=n+min(m,l)
 610  * @ if the first level idx-tbl store in a ascii idx-tbl.
 611  *   f(m,n,l)=1+min(m,l)
 612  */
 613 void * init_idx_tbl (SRC_FILE_DESC *pfiledesc)
 614 {
 615         /*
 616          * token string sorting
 617          * get row count and max word length.
 618          */
 619
 620         /*
 621          * create 128 unit for first char indexing.
 622          * for some state, there are few token defined,
 623          * it does not need first-char-indexding.
 624          */
 625
 626         /*
 627          * alloc memory accroding row count and max word length.
 628          * create index info for token strings.
 629          */
 630
 631         return NULL;
 632 }
 633 void * str_matching_by_idx_tbl (char *str, void *pvIdxTbl, struct token_id *pstTokenIdTbl, short *ptoken_len_tbl)
 634 {
 635         return NULL;
 636 }
 637 void init_token_len_tbl (const struct token_id *ptoken_tbl, unsigned short *plen_tbl, int size)
 638 {
 639         int i;
 640
 641         if (ptoken_tbl == NULL) return;
 642
 643         for (i=0; i<MAX_TOKEN_ID_TBL_SIZE; i++)
 644         {
 645                 if (ptoken_tbl[i].tokenstr == NULL &&
 646                         ptoken_tbl[i].token == 0 &&
 647                         ptoken_tbl[i].flags == 0 &&
 648                         ptoken_tbl[i].proc == NULL)
 649                         break;
 650
 651                 if (ptoken_tbl[i].name != NULL)
 652                 {
 653                         if (ptoken_tbl[i].chars[3] != 0)
 654                                 plen_tbl[i] = strlen(ptoken_tbl[i].name);
 655                         else
 656                                 plen_tbl[i] = strlen((char *)&ptoken_tbl[i].name);
 657                 }
 658
 659                 if (ptoken_tbl[i].chars[3] != 0)
 660                         INIT_PRINTF("%s(%d)\n", ptoken_tbl[i].name, plen_tbl[i]);
 661                 else
 662                         INIT_PRINTF("%s(%d)\n", (char *)&(ptoken_tbl[i].name), plen_tbl[i]);
 663         }
 664 }
 665
 666 void get_strlen_tbl (TOKEN_PROC_STATE_INFO *pstStateInfo)
 667 {
 668         int i;
 669         int size;
 670         const struct token_id *ptoken_tbl;
 671         short *plen_tbl;
 672
 673         if (pstStateInfo->strStateName == NULL) return;
 674         INIT_PRINTF("strStateName = %s;\n", pstStateInfo->strStateName);
 675
 676         ptoken_tbl = pstStateInfo->astTokenTypeTbl[TYPE_WORD].pstTokenTbl;
 677         INIT_PRINTF("[TYPE_WORD]: 0x%08x\n", (unsigned int)ptoken_tbl);
 678         if (ptoken_tbl != NULL)
 679         {
 680                 size = pstStateInfo->astTokenTypeTbl[TYPE_WORD].iTblSize;
 681                 plen_tbl = (short *) malloc(size*sizeof(short));
 682                 init_token_len_tbl(ptoken_tbl, plen_tbl, size);
 683                 pstStateInfo->api16TokenNameLen[TYPE_WORD] = plen_tbl;
 684         }
 685
 686         INIT_PRINTF("[TYPE_PUNCT]:\n");
 687         ptoken_tbl = pstStateInfo->astTokenTypeTbl[TYPE_PUNCT].pstTokenTbl;
 688         if (ptoken_tbl != NULL)
 689         {
 690                 size = pstStateInfo->astTokenTypeTbl[TYPE_PUNCT].iTblSize;
 691                 plen_tbl = (short *) malloc(size*sizeof(short));
 692                 init_token_len_tbl(ptoken_tbl, plen_tbl, size);
 693                 pstStateInfo->api16TokenNameLen[TYPE_PUNCT] = plen_tbl;
 694         }
 695
 696         INIT_PRINTF("[TYPE_PFX]:\n");
 697         ptoken_tbl = pstStateInfo->astTokenTypeTbl[TYPE_PFX].pstTokenTbl;
 698         if (ptoken_tbl != NULL)
 699         {
 700                 size = pstStateInfo->astTokenTypeTbl[TYPE_PFX].iTblSize;
 701                 plen_tbl = (short *) malloc(size*sizeof(short));
 702                 init_token_len_tbl(ptoken_tbl, plen_tbl, size);
 703                 pstStateInfo->api16TokenNameLen[TYPE_PFX] = plen_tbl;
 704         }
 705
 706         INIT_PRINTF("[TYPE_PAIR]:\n");
 707         ptoken_tbl = pstStateInfo->astTokenTypeTbl[TYPE_PAIR].pstTokenTbl;
 708         if (ptoken_tbl != NULL)
 709         {
 710                 size = pstStateInfo->astTokenTypeTbl[TYPE_PAIR].iTblSize;
 711                 plen_tbl = (short *) malloc(size*sizeof(short));
 712                 init_token_len_tbl(ptoken_tbl, plen_tbl, size);
 713                 pstStateInfo->api16TokenNameLen[TYPE_PAIR] = plen_tbl;
 714                 for (i=0; i<size; i++)
 715                 {
 716                         plen_tbl[i] >>= 1;
 717                 }
 718         }
 719
 720         INIT_PRINTF("[Micro]:\n");
 721         //ptoken_tbl = pstStateInfo->astTokenTypeTbl[TYPE_PAIR].pstTokenTbl;
 722         //size = pstStateInfo->astTokenTypeTbl[TYPE_PAIR].iTblSize;
 723         plen_tbl = (short *) malloc(size*sizeof(short));
 724         if (ptoken_tbl != NULL)
 725         {
 726                 init_token_len_tbl(macro_array, au16MacroTokenLen, ARRAY_SIZE(macro_array));
 727                 //pstStateInfo->api16TokenNameLen[TYPE_PAIR] = plen_tbl;
 728         }
 729 }
 730
 731 TOKEN_PROC_STATE *state_alloc_init (SRC_FILE_DESC *pfiledesc, int iState)
 732 {
 733         TOKEN_PROC_STATE *pstTokenState = (TOKEN_PROC_STATE *) malloc(sizeof(TOKEN_PROC_STATE));
 734         if (pstTokenState == NULL) return NULL;
 735
 736         INIT_PRINTF("state_alloc_init(%d);\n", iState);
 737         memset(pstTokenState, 0x00, sizeof(TOKEN_PROC_STATE));
 738
 739         pfiledesc->pstStateInfo = &pfiledesc->pstStateInfoTbl[iState];
 740         pstTokenState->pstStateInfo = pfiledesc->pstStateInfo;
 741         pstTokenState->iStateId = iState;
 742
 743         /* duplicate var-info to new state. */
 744         if (pfiledesc->pstCurrTokenState)
 745         {
 746                 pstTokenState->paired_token = pfiledesc->pstCurrTokenState->paired_token;
 747                 pstTokenState->state_token_str = pfiledesc->pstCurrTokenState->state_token_str;
 748
 749                 pstTokenState->token_str = pfiledesc->pstCurrTokenState->token_str;
 750                 pstTokenState->token_len = pfiledesc->pstCurrTokenState->token_len;
 751                 pfiledesc->pstCurrTokenState->paired_state = 0;
 752
 753                 if (pfiledesc->pstCurrTokenState->pfnPairTokenProc != NULL)
 754                         pstTokenState->pfnPairTokenProc = pfiledesc->pstCurrTokenState->pfnPairTokenProc;
 755                 pfiledesc->pstCurrTokenState->pfnPairTokenProc = NULL;
 756
 757                 if (pfiledesc->pstCurrTokenState->pfnTokenProc != NULL)
 758                         pstTokenState->pfnTokenProc = pfiledesc->pstCurrTokenState->pfnTokenProc;
 759                 pfiledesc->pstCurrTokenState->pfnTokenProc = NULL;
 760         }
 761         return pstTokenState;
 762 }
 763
 764 /*
 765  * fdesc: open file, and init SRC_FILE_DESC structure.
 766  */
 767 SRC_FILE_DESC *src_file_init (SRC_FILE_DESC *file, char *filename, TOKEN_PROC_STATE_INFO astTokenStateTbl[], int num)
 768 {
 769         if (filename != NULL)
 770         {
 771                 file->filename = filename;
 772                 file->file = fopen(filename, "r");
 773         }
 774         else
 775                 file->file = stdin;
 776
 777         file->pstStateInfoTbl = astTokenStateTbl;
 778         file->iStateInfoNum = num;
 779         file->pstCurrTokenState = state_alloc_init(file, STATE_CODE);
 780
 781         //file->pfnMTokenStrProc = NULL;
 782         file->pstMTokenTbl = macro_array;
 783         file->pu16MTokenNameLen = au16MacroTokenLen;
 784
 785         /* init len-tbl .api16TokenNameLen in TOKEN_PROC_STATE */
 786         for (int i; i<sizeof(gastInnerTokenStateTbl)/sizeof(gastInnerTokenStateTbl[0]); i++)
 787         {
 788                 INIT_PRINTF("########################################\n");
 789                 get_strlen_tbl(&file->pstStateInfoTbl[i]);
 790         }
 791
 792         file->lineno = 0;
 793
 794         file->iBlkSize = 0;
 795         file->iCurrIdx = 0;
 796
 797         /* do not need to init buffer */
 798 //      file->aBuff;
 799         file->bakchar = 0;
 800         file->type = 0;
 801         file->lasttype = 0;
 802         file->iTokenStrLen = 0;
 803         file->iPrevIdx = 0;
 804
 805         /* state stack */
 806         file->pStkBuff = malloc(MAX_STATE_NUM * sizeof(void *));
 807         QStackIF_InitStack(&file->stStateStk, file->pStkBuff, MAX_STATE_NUM * sizeof(void *));
 808         QStackIF_SetDataOprType(&file->stStateStk, OPR_UINT32);
 809
 810         /* token Q */
 811         file->pStkBuff = malloc(MAX_TOKEN_NUM * 2 * sizeof(void *));
 812         QStackIF_InitQueue(&file->stTokenQ, file->pStkBuff, MAX_TOKEN_NUM * 2 * sizeof(void *));
 813         QStackIF_SetDataOprType(&file->stTokenQ, OPR_UINT32);
 814
 815         return file;
 816 }
 817
 818 void src_file_release (SRC_FILE_DESC *file)
 819 {
 820         //
 821 }
 822
 823 int load_token_def_file (char *token_def_file)
 824 {
 825         int token;
 826         FILE *file;
 827         SRC_FILE_DESC src_code_file;
 828
 829         printf("open file: %s\n", token_def_file);
 830
 831         gpcurr_src_file = src_file_init (&src_code_file,
 832                                                                         token_def_file,
 833                                                                         gastInnerTokenStateTbl,
 834                                                                         ARRAY_SIZE(gastInnerTokenStateTbl));
 835
 836         while (token=token_read(gpcurr_src_file))
 837         {
 838 //              printf("Token = %d\n", token);
 839
 840                 on_token_proc(token);
 841         }
 842
 843         printf("token = %d\n", token);
 844
 845         return 0;
 846 }
 847
 848
 849
 850 /*******************************
 851  *******************************
 852  * End Of File
 853  *******************************
 854  *******************************/
 855
 856