ryzom/server/src/ai_service/script_compiler.cpp

   1 // Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
   2 // Copyright (C) 2010  Winch Gate Property Limited
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU Affero General Public License as
   6 // published by the Free Software Foundation, either version 3 of the
   7 // License, or (at your option) any later version.
   8 //
   9 // This program is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU Affero General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU Affero General Public License
  15 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 #include "stdpch.h"
  18
  19 #include "script_vm.h"
  20 #include "script_compiler.h"
  21
  22 // Compiler special keywords (tokens and rules)
  23 static std::string const s_kw_NUMBER       = "NUMBER";
  24 static std::string const s_kw_CHAIN        = "CHAIN";
  25 static std::string const s_kw_NAME         = "NAME";
  26 static std::string const s_kw_STRNAME      = "STRNAME";
  27 static std::string const s_kw_POINT        = "POINT";
  28 static std::string const s_kw_readConstVar = "readConstVar";
  29 static std::string const s_kw_lineOrClose  = "lineOrClose";
  30 static std::string const s_kw_params       = "params";
  31 static std::string const s_kw_tuple        = "tuple";
  32 static std::string const s_kw_expeclose    = "expeclose";
  33 static std::string const s_kw_exp          = "exp";
  34 static std::string const s_kw_somme        = "somme";
  35 static std::string const s_kw_produit      = "produit";
  36 static std::string const s_kw_facteur      = "facteur";
  37 static std::string const s_kw_case         = "case";
  38
  39 using namespace std;
  40 using namespace NLMISC;
  41
  42 //////////////////////////////////////////////////////////////////////////
  43 // A Small Custom Compiler For AI.
  44 // (Token and Grammar are Upgradable, Error returns have to be upgraded).
  45 // Ask to Stephane Le Dorze for Explanations.
  46 //////////////////////////////////////////////////////////////////////////
  47
  48 namespace AICOMP
  49 {
  50
  51
  52 /****************************************************************************/
  53 /* Script related classes                                                   */
  54 /****************************************************************************/
  55
  56 //////////////////////////////////////////////////////////////////////////////
  57 // CScriptNativeFuncParams                                                  //
  58 //////////////////////////////////////////////////////////////////////////////
  59
  60 CScriptNativeFuncParams::CScriptNativeFuncParams(const std::string &str, FScrptNativeFunc func)
  61         : _func(func)
  62         , _va(false)
  63 {
  64         const size_t lastPartIndex2=str.find_last_of("_", string::npos);
  65         nlassert(lastPartIndex2!=0 && lastPartIndex2!=string::npos);
  66         const size_t lastPartIndex=str.find_last_of("_", lastPartIndex2-1);
  67         nlassert(lastPartIndex!=0 && lastPartIndex!=string::npos);
  68
  69         _nbInParams=lastPartIndex2-lastPartIndex-1;
  70         _nbOutParams=str.size()-lastPartIndex2-1;
  71         if (_nbInParams==1 && str[lastPartIndex+1] == 'v')
  72         {
  73                 _nbInParams = ~0;
  74                 _va = true;
  75         }
  76         if (_nbOutParams==1 && str[lastPartIndex2+1] == 'v')
  77         {
  78                 _nbOutParams = ~0;
  79                 _va = true;
  80         }
  81 }
  82
  83 //////////////////////////////////////////////////////////////////////////////
  84 // CJumpRememberer                                                          //
  85 //////////////////////////////////////////////////////////////////////////////
  86
  87 class CJumpRememberer
  88 {
  89 public:
  90         CJumpRememberer(size_t codeBlockIndex);
  91         size_t _where;
  92         size_t _codeBlockIndex;
  93 };
  94
  95 //////////////////////////////////////////////////////////////////////////////
  96 // CJumpTable                                                               //
  97 //////////////////////////////////////////////////////////////////////////////
  98
  99 class CJumpTable
 100 {
 101 public:
 102         CJumpTable(const CSmartPtr<AIVM::CByteCode> &byteCode);
 103         virtual ~CJumpTable();
 104         void add(CJumpRememberer jump);
 105         void newCodeBlock();
 106 private:
 107         vector<size_t> _codeOffsets;
 108         vector<CJumpRememberer> _jumps;
 109         CSmartPtr<AIVM::CByteCode> _byteCode;
 110 };
 111
 112 //////////////////////////////////////////////////////////////////////////////
 113 // CCaseTracer                                                              //
 114 //////////////////////////////////////////////////////////////////////////////
 115
 116 class CCaseTracer : public CRefCount
 117 {
 118 public:
 119         CCaseTracer(const CSmartPtr<CSubRuleTracer> &tracer, const string &sourceName);
 120
 121         CSmartPtr<CSubRuleTracer> _tracer;
 122         CSmartPtr<const AIVM::CByteCode> _code;
 123         size_t _sortValue;
 124 };
 125
 126 /****************************************************************************/
 127 /* TOKEN SPECIALIZATION                                                     */
 128 /****************************************************************************/
 129
 130 /* used:
 131         []              -> or for character.
 132         -               -> from before to after (char) (inside []).
 133         "c"             -> character c (or \c)
 134
 135         {xx}    -> the token 'xx'.
 136         ()              -> enclosing
 137
 138         |               -> or
 139
 140         +               -> card suffixe 1..n
 141         *               -> card suffixe 0..n
 142         ?               -> card suffixe 0..1
 143         {m,n}   -> card suffixe m..n
 144 */
 145
 146 //////////////////////////////////////////////////////////////////////////////
 147 // CBracketToken                                                            //
 148 //////////////////////////////////////////////////////////////////////////////
 149
 150 // [a-zA-Z0-9] (readOK)
 151 class CBracketToken : public CBasicToken
 152 {
 153 public:
 154         CBasicToken *createNew() const;
 155         size_t init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex);
 156         CTokenTestResult buildNode(const std::string &code, size_t &index) const;
 157         void dump(size_t indent) const;
 158
 159 private:
 160         string _Body;
 161 };
 162
 163 //////////////////////////////////////////////////////////////////////////////
 164 // CCharToken                                                               //
 165 //////////////////////////////////////////////////////////////////////////////
 166
 167 // "c"
 168 class CCharToken : public CBasicToken
 169 {
 170 public:
 171         CBasicToken *createNew() const;
 172         size_t init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex);
 173         CTokenTestResult buildNode(const std::string &code, size_t &index) const;
 174         void dump(size_t indent) const;
 175
 176 private:
 177         char _c;
 178 };
 179
 180 //////////////////////////////////////////////////////////////////////////////
 181 // CTokenToken                                                              //
 182 //////////////////////////////////////////////////////////////////////////////
 183
 184 // {mytoken}
 185 class CTokenToken : public CBasicToken
 186 {
 187 public:
 188         CBasicToken *createNew() const;
 189         size_t init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex);
 190         CTokenTestResult buildNode(const std::string &code, size_t &index) const;
 191         void dump(size_t indent) const;
 192
 193 private:
 194         string _tokenName;
 195 };
 196
 197 //////////////////////////////////////////////////////////////////////////////
 198 // CParenthesisToken                                                        //
 199 //////////////////////////////////////////////////////////////////////////////
 200
 201 // (..)
 202 class CParenthesisToken : public CBasicToken
 203 {
 204 public:
 205         CBasicToken *createNew() const;
 206         size_t init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex);
 207         CTokenTestResult buildNode(const std::string &code, size_t &index) const;
 208         void dump(size_t indent) const;
 209
 210 private:
 211         TTokenList _tokenList;
 212 };
 213
 214 //////////////////////////////////////////////////////////////////////////////
 215 // COrToken                                                                 //
 216 //////////////////////////////////////////////////////////////////////////////
 217
 218 // | (readOK)
 219 class COrToken : public CBasicToken
 220 {
 221 public:
 222         CBasicToken *createNew() const;
 223         size_t init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex);
 224         CTokenTestResult buildNode(const std::string &code, size_t &index) const;
 225         void dump(size_t indent) const;
 226
 227 private:
 228         CSmartPtr<CBasicToken> firstToken;
 229         CSmartPtr<CBasicToken> secondToken;
 230 };
 231
 232 //////////////////////////////////////////////////////////////////////////////
 233 // CCardToken                                                               //
 234 //////////////////////////////////////////////////////////////////////////////
 235
 236 // * + ? (readOK)
 237 class CCardToken : public CBasicToken
 238 {
 239 public:
 240         enum TCardType
 241         {
 242                 CARD_ZERO_ONE=0,
 243                 CARD_ZERO_MANY,
 244                 CARD_ONE_MANY,
 245         };
 246
 247         CBasicToken *createNew() const;
 248         size_t init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex);
 249         CTokenTestResult buildNode(const std::string &code, size_t &index) const;
 250         void dump(size_t indent) const;
 251
 252 private:
 253         CSmartPtr<CBasicToken> _childToken;
 254         TCardType _card;
 255 };
 256
 257 /****************************************************************************/
 258 /** Script related classes implementation ***********************************/
 259 /****************************************************************************/
 260
 261 /****************************************************************************/
 262 /* Tokens                                                                   */
 263 /****************************************************************************/
 264
 265 //////////////////////////////////////////////////////////////////////////////
 266 // CBracketToken implementation                                             //
 267 //////////////////////////////////////////////////////////////////////////////
 268
 269 CBasicToken *CBracketToken::createNew() const
 270 {
 271         return new CBracketToken();
 272 }
 273
 274 size_t CBracketToken::init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 275 {
 276         const size_t index = str.find_first_of("]", firstIndex);
 277         nlassert(index!=string::npos);
 278         firstIndex++; // pass '['
 279         _Body = str.substr(firstIndex, index-firstIndex);
 280         tokenList.push_back(this);
 281         return index+1;
 282 }
 283
 284 CTokenTestResult CBracketToken::buildNode(const std::string &code, size_t &index) const
 285 {
 286         if (index>=code.size())
 287                 return CTokenTestResult(NULL, CTokenTestResult::BRULE_INVALID);
 288
 289         const char c = code.at(index);
 290         bool success = false;
 291         bool first = true;
 292         FOREACHC(p, string, _Body)
 293         {
 294                 const char cp = *p;
 295                 if (first)
 296                 {
 297                         if (cp==c)
 298                                 goto found;
 299                         first=false;
 300                         continue;
 301                 }
 302
 303                 if (cp=='-')
 304                 {
 305                         if ( c>=*(p-1)
 306                                 && c<=*(p+1))
 307                                 goto found;
 308                         p++; // to pass the last letter of the range
 309                         continue;
 310                 }
 311
 312                 if (cp==c)
 313                         goto found;
 314         }
 315
 316         return CTokenTestResult(NULL, CTokenTestResult::BRULE_INVALID);
 317 found:
 318         index++;
 319         string name;
 320         name+=c;
 321         CSmartPtr<CCodeNode> codeNode=new CCodeNode("Bracket",name);
 322         return CTokenTestResult(codeNode);
 323 }
 324
 325 void CBracketToken::dump(size_t indent) const
 326 {
 327         string str;
 328         str.resize(indent,' ');
 329         str+="["+_Body+"]";
 330         nldebug(str.c_str());
 331 }
 332
 333 //////////////////////////////////////////////////////////////////////////////
 334 // CCharToken implementation                                                //
 335 //////////////////////////////////////////////////////////////////////////////
 336
 337 CBasicToken *CCharToken::createNew() const
 338 {
 339         return new CCharToken();
 340 }
 341
 342 size_t CCharToken::init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 343 {
 344         if (str.at(firstIndex)=='\\')
 345         {
 346                 _c=str.at(firstIndex+1);
 347                 tokenList.push_back(this);
 348                 return firstIndex+2;
 349         }
 350         else
 351         {
 352                 nlassert(str.at(firstIndex)=='"');
 353                 nlassert(str.at(firstIndex+2)=='"');
 354                 _c=str.at(firstIndex+1);
 355                 tokenList.push_back(this);
 356                 return firstIndex+3;
 357         }
 358 }
 359
 360 CTokenTestResult CCharToken::buildNode(const std::string &code, size_t &index) const
 361 {
 362         if ( index<code.size()
 363                 && code.at(index)==_c)
 364         {
 365                 index++;
 366                 string name;
 367                 name+=_c;
 368
 369                 CSmartPtr<CCodeNode> codeNode=new CCodeNode("Char",name);
 370                 return CTokenTestResult(codeNode);
 371         }
 372         return CTokenTestResult(NULL, CTokenTestResult::BRULE_INVALID);
 373 }
 374
 375 void CCharToken::dump(size_t indent) const
 376 {
 377         string str;
 378         str.resize(indent,' ');
 379         str+="'";
 380         str+=_c;
 381         str+="'";
 382         nldebug(str.c_str());
 383 }
 384
 385 //////////////////////////////////////////////////////////////////////////////
 386 // CTokenToken implementation                                               //
 387 //////////////////////////////////////////////////////////////////////////////
 388
 389 CBasicToken *CTokenToken::createNew() const
 390 {
 391         return new CTokenToken();
 392 }
 393
 394 size_t CTokenToken::init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 395 {
 396         size_t index=str.find_first_of("}", firstIndex);
 397         nlassert(index!=string::npos);
 398         firstIndex++;
 399         _tokenName=str.substr(firstIndex, index-firstIndex);
 400         tokenList.push_back(this);
 401         return index+1;
 402 }
 403
 404 CTokenTestResult CTokenToken::buildNode(const std::string &code, size_t &index) const
 405 {
 406         CToken *const token=CCompiler::getInstance().getToken(_tokenName);
 407         nlassert(token!=NULL);
 408         const size_t lastIndex=index;
 409         CTokenTestResult res=token->buildTree(code,index);
 410         if (!res.isValid())
 411         {
 412                 nlwarning("token %s not succeed index %d", _tokenName.c_str(), index);
 413                 return res;
 414         }
 415
 416         CSmartPtr<CCodeNode> codeNode=new CCodeTokenNode("token", _tokenName, res.getCode());
 417         return CTokenTestResult(codeNode);
 418 }
 419
 420 void CTokenToken::dump(size_t indent) const
 421 {
 422         string str;
 423         str.resize(indent,' ');
 424         str+="{"+_tokenName+"}";
 425         nldebug(str.c_str());
 426 }
 427
 428 //////////////////////////////////////////////////////////////////////////////
 429 // CParenthesisToken implementation                                         //
 430 //////////////////////////////////////////////////////////////////////////////
 431
 432 CBasicToken *CParenthesisToken::createNew() const
 433 {
 434         return new CParenthesisToken();
 435 }
 436
 437 size_t CParenthesisToken::init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 438 {
 439         const size_t index=initTokens(_tokenList, str, firstIndex+1, lastIndex);
 440         nlassert(str.at(index)==')');
 441         tokenList.push_back(this);
 442         return index+1;
 443 }
 444
 445 CTokenTestResult CParenthesisToken::buildNode(const std::string &code, size_t &index) const
 446 {
 447         CSmartPtr<CCodeNode> codeNode;
 448         CSmartPtr<CCodeNode> nextCodeNode;
 449         size_t localIndex=index;
 450
 451         FOREACHC(tokenIt, TTokenList, _tokenList)
 452         {
 453                 CTokenTestResult res=(*tokenIt)->buildNode(code, localIndex);
 454                 if (!res.isValid())
 455                         return CTokenTestResult(NULL, CTokenTestResult::BRULE_INVALID);
 456
 457                 CSmartPtr<CCodeNode> localCodeNode=res.getCode();
 458                 if (!localCodeNode)
 459                         continue;
 460
 461                 // Chain result node.
 462                 if (!codeNode)
 463                         codeNode=localCodeNode;
 464                 else
 465                         nextCodeNode->_NextNode=localCodeNode;
 466                 nextCodeNode=localCodeNode;
 467                 while (nextCodeNode->_NextNode)
 468                         nextCodeNode=nextCodeNode->_NextNode;
 469         }
 470         if (codeNode)
 471                 index=localIndex;
 472         return codeNode;
 473 }
 474
 475 void CParenthesisToken::dump(size_t indent) const
 476 {
 477         string str;
 478         str.resize(indent,' ');
 479
 480         nldebug((str+"(").c_str());
 481         FOREACHC(tokenIt,TTokenList,_tokenList)
 482                 (*tokenIt)->dump(indent+1);
 483         nldebug((str+")").c_str());
 484 }
 485
 486 //////////////////////////////////////////////////////////////////////////////
 487 // COrToken implementation                                                  //
 488 //////////////////////////////////////////////////////////////////////////////
 489
 490 CBasicToken *COrToken::createNew() const
 491 {
 492         return new COrToken();
 493 }
 494
 495 size_t COrToken::init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 496 {
 497         nlassert(tokenList.size()>0);
 498         const size_t orIndex=tokenList.size()-1;
 499         const size_t finalIndex=initTokens(tokenList, str, firstIndex+1, lastIndex);
 500         nlassert(tokenList.size()>1);
 501
 502         // insert the or operation.
 503         firstToken=tokenList[orIndex];
 504         secondToken=tokenList[orIndex+1];
 505         tokenList[orIndex]=this;
 506         tokenList.erase(tokenList.begin()+orIndex+1);
 507
 508         return finalIndex;
 509 }
 510
 511 CTokenTestResult COrToken::buildNode(const std::string &code, size_t &index) const
 512 {
 513         size_t firstIndex=index;
 514         CTokenTestResult firstRes=firstToken->buildNode         (code, firstIndex);
 515         if (!firstRes.isValid())
 516         {
 517                 size_t secondIndex=index;
 518                 CTokenTestResult secondRes=secondToken->buildNode       (code, secondIndex);
 519                 index=secondIndex;
 520                 return secondRes;
 521         }
 522
 523         index=firstIndex;
 524         return firstRes;
 525 }
 526
 527 void COrToken::dump(size_t indent) const
 528 {
 529         firstToken->dump(indent+1);
 530         string str;
 531         str.resize(indent,' ');
 532         str+="|";
 533         nldebug(str.c_str());
 534         secondToken->dump(indent+1);
 535 }
 536
 537 //////////////////////////////////////////////////////////////////////////////
 538 // CCardToken implementation                                                //
 539 //////////////////////////////////////////////////////////////////////////////
 540
 541 CBasicToken *CCardToken::createNew() const
 542 {
 543         return new CCardToken();
 544 }
 545
 546 size_t CCardToken::init(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 547 {
 548         nlassert(tokenList.size()>0);
 549
 550         switch(str.at(firstIndex))
 551         {
 552         case '?':
 553                 _card = CARD_ZERO_ONE;
 554                 break;
 555         case '*':
 556                 _card = CARD_ZERO_MANY;
 557                 break;
 558         case '+':
 559                 _card = CARD_ONE_MANY;
 560                 break;
 561         default:
 562                 break;
 563         }
 564
 565         _childToken = tokenList.back();
 566         tokenList.back()=this;
 567
 568         return firstIndex+1;
 569 }
 570
 571 CTokenTestResult CCardToken::buildNode(const std::string &code, size_t &index) const
 572 {
 573         CSmartPtr<CCodeNode> firstCodeNode = NULL;
 574         CSmartPtr<CCodeNode> nextCodeNode = NULL;
 575
 576         size_t localIndex = index;
 577         size_t nbNodes = 0;
 578
 579         CTokenTestResult res;
 580
 581         while ((res=_childToken->buildNode(code, localIndex)).isValid())
 582         {
 583                 CSmartPtr<CCodeNode> newCodeNode=res.getCode();
 584                 if (!newCodeNode)
 585                         continue;
 586
 587                 if (_card==CARD_ZERO_ONE)
 588                 {
 589                         index=localIndex;
 590                         return res;
 591                 }
 592
 593                 if (!firstCodeNode)
 594                         firstCodeNode = newCodeNode;
 595                 else
 596                         nextCodeNode->_NextNode=newCodeNode;
 597                 nextCodeNode=newCodeNode;
 598                 while (nextCodeNode->_NextNode)
 599                         nextCodeNode = nextCodeNode->_NextNode;
 600                 nbNodes++;
 601         }
 602         if (firstCodeNode)
 603                 index = localIndex;
 604         return CTokenTestResult(firstCodeNode, (nbNodes==0 && _card==CARD_ONE_MANY)?CTokenTestResult::BRULE_INVALID:CTokenTestResult::BRULE_VALID);
 605 }
 606
 607 void CCardToken::dump(size_t indent) const
 608 {
 609         string str;
 610         str.resize(indent,' ');
 611         switch(_card)
 612         {
 613         case CARD_ZERO_ONE:
 614                 str += "?(0..1)";
 615                 break;
 616         case CARD_ZERO_MANY:
 617                 str += "*(0..n)";
 618                 break;
 619         case CARD_ONE_MANY:
 620                 str += "+(1..n)";
 621                 break;
 622         default:
 623                 break;
 624         }
 625         nldebug(str.c_str());
 626         _childToken->dump(indent+1);
 627 }
 628
 629 /****************************************************************************/
 630 /* Nodes                                                                    */
 631 /****************************************************************************/
 632
 633 //////////////////////////////////////////////////////////////////////////////
 634 // CCodeNode implementation                                                 //
 635 //////////////////////////////////////////////////////////////////////////////
 636
 637 CCodeNode::CCodeNode(const string &type, const string &name, CSmartPtr<CCodeNode> firstChildNode)
 638 : _Type(type)
 639 , _Name(name)
 640 , _FirstChildNode(firstChildNode)
 641 {
 642 }
 643
 644 void CCodeNode::dump(size_t indent)
 645 {
 646         string str;
 647         str.resize(indent,' ');
 648         str+=_Type+":"+_Name;
 649         //              nlwarning(str.c_str());
 650         if (_NextNode)
 651                 _NextNode->dump(indent);
 652 }
 653
 654 string CCodeNode::getFullName() const
 655 {
 656         if (_NextNode)
 657                 return _Name+_NextNode->getFullName();
 658         return _Name;
 659 }
 660
 661 //////////////////////////////////////////////////////////////////////////////
 662 // CCodeTokenNode implementation                                            //
 663 //////////////////////////////////////////////////////////////////////////////
 664
 665 CCodeTokenNode::CCodeTokenNode(const string &type, const string &name, CSmartPtr<CCodeNode> firstChildNode)
 666 : CCodeNode(type, name, firstChildNode)
 667 {
 668 }
 669
 670 void CCodeTokenNode::dump(size_t indent)
 671 {
 672         string str;
 673         str.resize(indent,' ');
 674         str+=_Type+":"+_Name;
 675
 676         if (_FirstChildNode)
 677                 str+=":"+_FirstChildNode->getFullName();
 678
 679         nldebug(str.c_str());
 680         if (_FirstChildNode)
 681                 _FirstChildNode->dump(indent+1);
 682
 683         if (_NextNode)
 684                 _NextNode->dump(indent);
 685 }
 686
 687 string CCodeTokenNode::getFullName() const
 688 {
 689         string returnName;
 690         if (_FirstChildNode)
 691                 returnName+=_FirstChildNode->getFullName();
 692         if (_NextNode)
 693                 returnName+=_NextNode->getFullName();
 694         return returnName;
 695 }
 696
 697 /****************************************************************************/
 698 /*                                                                          */
 699 /****************************************************************************/
 700
 701 /*
 702  * Lexx
 703         x        the character "x"
 704         "x"      an "x", even if x is an operator.
 705         \x       an "x", even if x is an operator.
 706         [xy]     the character x or y.
 707         [x-z]    the characters x, y or z.
 708         [^x]     any character but x.
 709         .        any character but newline.
 710         ^x       an x at the beginning of a line.
 711         <y>x     an x when Lex is in start condition y.
 712         x$       an x at the end of a line.
 713         x?       an optional x.
 714         x*       0,1,2, ... instances of x.
 715         x+       1,2,3, ... instances of x.
 716         x|y      an x or a y.
 717         (x)      an x.
 718         x/y      an x but only if followed by y.
 719         {xx}     the translation of xx from the
 720         definitions section.
 721         x{m,n}   m through n occurrences of x
 722 */
 723
 724 /* used:
 725         -               -> from before to after (char)
 726         []              -> or for character.
 727         \c              -> character c
 728         "c"             -> character c
 729         {token} -> match token ?
 730         ()              -> enclosing
 731         |               -> or
 732         +               -> card suffixe 1..n
 733         *               -> card suffixe 0..n
 734         ?               -> card suffixe 0..1
 735 //      {m,n}   -> card suffixe         m..n
 736 */
 737
 738
 739 //////////////////////////////////////////////////////////////////////////////
 740 // CRule implementation                                                     //
 741 //////////////////////////////////////////////////////////////////////////////
 742
 743 CRule::CRule(const std::string &name, const string &decl)
 744 : _Name(name)
 745 {
 746         setDesc(decl);
 747 }
 748
 749 void CRule::setDesc(const string &decl)
 750 {
 751         const AIVM::CStringSeparator strSep(decl, "|");
 752         while (strSep.hasNext())
 753                 _subRules.push_back(new CSubRule(this,strSep.get()));
 754 }
 755
 756 //////////////////////////////////////////////////////////////////////////////
 757 // CSubRule implementation                                                  //
 758 //////////////////////////////////////////////////////////////////////////////
 759
 760 CSubRule::CSubRule(CRule *parent, const  string &decl)
 761         :_Parent(parent)
 762 {
 763         const AIVM::CStringSeparator strSepTokenAction(decl, ",");
 764
 765         // Tokens.
 766         if (strSepTokenAction.hasNext())
 767         {
 768                 const string Tokens=strSepTokenAction.get();
 769
 770                 // Tokens.
 771                 const AIVM::CStringSeparator strSep(Tokens, " ");
 772                 while (strSep.hasNext())
 773                         _tokens.push_back(strSep.get());
 774         }
 775
 776         // Action.
 777         while (strSepTokenAction.hasNext())
 778                 _ExecOpCodes.push_back(strSepTokenAction.get());
 779
 780         nlassert(_tokens.size()>0);
 781 }
 782
 783 //////////////////////////////////////////////////////////////////////////////
 784 // CBasicToken implementation                                               //
 785 //////////////////////////////////////////////////////////////////////////////
 786
 787 size_t CBasicToken::initTokens(TTokenList &tokenList, const string &str, size_t firstIndex, size_t lastIndex)
 788 {
 789         size_t index=firstIndex;
 790         while (index<lastIndex)
 791         {
 792                 CSmartPtr<CBasicToken> token=getNewToken(str.at(index));
 793                 if (!token)
 794                         break;
 795                 index=token->init(tokenList, str, index, lastIndex);
 796         }
 797         return index;
 798 }
 799
 800 CSmartPtr<CBasicToken> CBasicToken::getNewToken(char c)
 801 {
 802         if (_BasicTokens.empty())
 803         {
 804                 insertBasicToken('[', new CBracketToken());
 805
 806                 insertBasicToken('"', new CCharToken());
 807                 insertBasicToken('\\', new CCharToken());
 808
 809                 insertBasicToken('{', new CTokenToken());
 810                 insertBasicToken('(', new CParenthesisToken());
 811                 insertBasicToken('|', new COrToken());
 812
 813                 insertBasicToken('*', new CCardToken());
 814                 insertBasicToken('?', new CCardToken());
 815                 insertBasicToken('+', new CCardToken());
 816         }
 817         TBasicTokenList::iterator it=_BasicTokens.find(c);
 818         if (it==_BasicTokens.end())
 819                 return NULL;
 820         return it->second->createNew();
 821 }
 822
 823 void CBasicToken::insertBasicToken(char id, CSmartPtr<CBasicToken> token)
 824 {
 825         _BasicTokens.insert(make_pair(id,token));
 826 }
 827
 828
 829 //////////////////////////////////////////////////////////////////////////////
 830 // CToken implementation                                                    //
 831 //////////////////////////////////////////////////////////////////////////////
 832
 833 CToken::CToken(const  string    &tokenName, const string &tokenDesc)
 834         :_tokenName(tokenName), _tokenDesc(tokenDesc)
 835 {
 836         const size_t index = CBasicToken::initTokens(_Tokens, _tokenDesc, 0, _tokenDesc.size());
 837         nlassert(index==tokenDesc.size());
 838 }
 839
 840 void CToken::dump() const
 841 {
 842         const string str = "Token:"+_tokenName+" : "+_tokenDesc;
 843         nldebug(str.c_str());
 844         FOREACHC(tokenIt,TTokenContainer,_Tokens)
 845                 (*tokenIt)->dump(0);
 846 }
 847
 848 CTokenTestResult CToken::buildTree(const std::string &code, size_t &index)
 849 {
 850         CSmartPtr<CCodeNode> masterNode;
 851         CSmartPtr<CCodeNode> currentNode;
 852         size_t  localIndex = index;
 853
 854         FOREACHC(tokenIt,std::vector<CSmartPtr<CBasicToken> >,_Tokens)
 855         {
 856                 CTokenTestResult res = (*tokenIt)->buildNode(code,localIndex);
 857
 858                 if (!res.isValid())
 859                         return CTokenTestResult(NULL, CTokenTestResult::BRULE_INVALID);
 860
 861                 CSmartPtr<CCodeNode> const newCodeNode = res.getCode();
 862                 if (!newCodeNode)
 863                         continue;
 864
 865                 // Chain result node.
 866                 if (!masterNode)
 867                         masterNode=newCodeNode;
 868                 else
 869                         currentNode->_NextNode=newCodeNode;
 870                 currentNode=newCodeNode;
 871                 while (currentNode->_NextNode)
 872                         currentNode=currentNode->_NextNode;
 873         }
 874
 875         if (masterNode)
 876                 index=localIndex;
 877         return masterNode;
 878 }
 879
 880 const std::string &CToken::getName() const
 881 {
 882         return _tokenName;
 883 }
 884
 885 //////////////////////////////////////////////////////////////////////////////
 886 // CCompiler implementation                                                 //
 887 //////////////////////////////////////////////////////////////////////////////
 888
 889 CCompiler::CCompiler()
 890 {
 891         string cfgFile = NLMISC::CPath::lookup ("ais_script_compiler.cfg", false);
 892         if (!cfgFile.empty ())
 893         {
 894                 NLMISC::CIFile file(cfgFile);
 895                 const int bufferSize = 256;
 896                 char buffer[bufferSize];
 897
 898                 while (!file.eof())
 899                 {
 900                         file.getline(buffer, bufferSize);
 901                         if (buffer[0]=='#' || buffer[0]=='\0') // Skip lines beginning with a # and empty lines
 902                                 continue;
 903                         string line = buffer;
 904                         const string sep1 = ": ";
 905                         const string sep2 = "=";
 906                         string part1, part2, part3;
 907
 908                         string::size_type pos1 = line.find(sep1);
 909                         if (pos1!=string::npos)
 910                         {
 911                                 pos1+=sep1.size();
 912                                 string::size_type pos2 = line.find(sep2, pos1);
 913                                 if (pos2!=string::npos)
 914                                 {
 915                                         pos2+=sep2.size();
 916                                         if (pos1!=string::npos && pos2!=string::npos)
 917                                         {
 918                                                 part1 = line.substr(0, pos1-sep1.size()); // begin to sep1
 919                                                 part2 = line.substr(pos1, pos2-pos1-sep2.size()); // sep1 to sep2
 920                                                 part3 = line.substr(pos2); // sep2 to end
 921                                         }
 922                                 }
 923                         }
 924
 925                         if (part1.size()!=0 && part2.size()!=0 && part3.size()!=0)
 926                         {
 927                                 if (part1=="token")
 928                                 {
 929                                         addToken(part2, part3);
 930                                 }
 931                                 else if (part1=="rule")
 932                                 {
 933                                         addRule(part2, part3);
 934                                 }
 935                         }
 936                         else
 937                         {
 938                                 nlwarning("Invalid script line: \"%s\"", line.c_str());
 939                         }
 940                 }
 941
 942         // Basic Opcodes ---------------------------------------------------------
 943 #define REGISTER_OPCODE(__opcode) addOpcode(#__opcode,::AIVM::CScriptVM::__opcode)
 944
 945         REGISTER_OPCODE(AND);
 946         REGISTER_OPCODE(OR);
 947         REGISTER_OPCODE(NOT);
 948
 949         REGISTER_OPCODE(EQ);
 950         REGISTER_OPCODE(NEQ);
 951         REGISTER_OPCODE(INF);
 952         REGISTER_OPCODE(INFEQ);
 953         REGISTER_OPCODE(SUP);
 954         REGISTER_OPCODE(SUPEQ);
 955
 956         REGISTER_OPCODE(ADD);
 957         REGISTER_OPCODE(SUB);
 958         REGISTER_OPCODE(MUL);
 959         REGISTER_OPCODE(DIV);
 960
 961         REGISTER_OPCODE(PUSH_ON_STACK);
 962         REGISTER_OPCODE(POP);
 963
 964         REGISTER_OPCODE(SET_VAR_VAL);
 965         REGISTER_OPCODE(SET_STR_VAR_VAL);
 966         REGISTER_OPCODE(SET_CTX_VAR_VAL);
 967
 968         REGISTER_OPCODE(PUSH_VAR_VAL);
 969         REGISTER_OPCODE(PUSH_STR_VAR_VAL);
 970         REGISTER_OPCODE(PUSH_CTX_VAR_VAL);
 971
 972 //      REGISTER_OPCODE(SET_OTHER_VAR_VAL);
 973 //      REGISTER_OPCODE(SET_OTHER_STR_VAR_VAL);
 974 //      REGISTER_OPCODE(SET_OTHER_CTX_VAR_VAL);
 975
 976 //      REGISTER_OPCODE(PUSH_OTHER_VAR_VAL);
 977 //      REGISTER_OPCODE(PUSH_OTHER_STR_VAR_VAL);
 978 //      REGISTER_OPCODE(PUSH_OTHER_CTX_VAR_VAL);
 979
 980         REGISTER_OPCODE(SET_CONTEXT_VAR_VAL);
 981         REGISTER_OPCODE(SET_CONTEXT_STR_VAR_VAL);
 982         REGISTER_OPCODE(SET_CONTEXT_CTX_VAR_VAL);
 983
 984         REGISTER_OPCODE(PUSH_CONTEXT_VAR_VAL);
 985         REGISTER_OPCODE(PUSH_CONTEXT_STR_VAR_VAL);
 986         REGISTER_OPCODE(PUSH_CONTEXT_CTX_VAR_VAL);
 987
 988         REGISTER_OPCODE(JUMP);
 989         REGISTER_OPCODE(JE);
 990         REGISTER_OPCODE(JNE);
 991
 992         REGISTER_OPCODE(PUSH_PRINT_STRING);
 993         REGISTER_OPCODE(PUSH_PRINT_VAR);
 994         REGISTER_OPCODE(PUSH_PRINT_STR_VAR);
 995         REGISTER_OPCODE(PRINT_STRING);
 996
 997         REGISTER_OPCODE(LOG_STRING);
 998
 999         REGISTER_OPCODE(FUNCTION);
1000         REGISTER_OPCODE(CALL);
1001
1002         REGISTER_OPCODE(PUSH_THIS);
1003         REGISTER_OPCODE(PUSH_GROUP);
1004
1005         REGISTER_OPCODE(PUSH_STRING);
1006         REGISTER_OPCODE(ASSIGN_FUNC_FROM);
1007
1008         REGISTER_OPCODE(NATIVE_CALL);
1009         REGISTER_OPCODE(RAND);
1010         REGISTER_OPCODE(RANDEND);
1011
1012         REGISTER_OPCODE(RET);
1013
1014         REGISTER_OPCODE(EOP);
1015
1016         REGISTER_OPCODE(ONCHILDREN);
1017         REGISTER_OPCODE(SWITCH);
1018
1019         REGISTER_OPCODE(INCR);
1020         REGISTER_OPCODE(DECR);
1021
1022         REGISTER_OPCODE(CONCAT);
1023         REGISTER_OPCODE(FTOS);
1024         }
1025
1026         // Natives Funcs ---------------------------------------------------------
1027
1028         registerNativeFunc();
1029 }
1030
1031 CSmartPtr<CSubRuleTracer> CCompiler::buildCodeTree (const string &code) const
1032 {
1033         size_t tokenIndex=0;
1034         size_t index=0;
1035
1036         CSmartPtr<CSubRuleTracer> lastInsertedTracer;
1037         CSmartPtr<CSubRuleTracer> firstInsertedTracer;
1038
1039         // For each token of the code.
1040         while (index<code.size())
1041         {
1042                 std::string tokenName;
1043                 std::string textValue;
1044                 const size_t lastIndex=index;
1045                 if (!getNextToken(code, index, tokenName, textValue))
1046                         break;
1047
1048                 tokenIndex++;
1049
1050 #ifdef DISPLAY_INFOS
1051                 {
1052                         string str(">> TOKEN: ");
1053                         str+=tokenName;
1054                         str+=" "+toString(tokenIndex);
1055                         nldebug(str.c_str());
1056                 }
1057 #endif
1058                 CSmartPtr<CSubRuleTracer> tracer=new CSubRuleTracer(lastIndex, index-1, tokenName, textValue);
1059
1060                 if (lastInsertedTracer.isNull())
1061                 {
1062                         lastInsertedTracer=tracer;
1063                         firstInsertedTracer=tracer;
1064                 }
1065                 else
1066                 {
1067                         tracer->checkRules(index-1);
1068                         lastInsertedTracer=tracer;
1069                 }
1070         }
1071         return firstInsertedTracer;
1072 }
1073
1074
1075 CSmartPtr<const AIVM::CByteCode> CCompiler::compileCode (const std::vector<std::string> &sourceCodeLines, const string &fullName) const
1076 {
1077         CSubRuleTracer::_PreviousTracers.clear();
1078         CSubRuleTracer::_NextTracers.clear();
1079
1080         typedef const std::vector<std::string> TList; // because there a problem with const in the macro.
1081         string code="{ \n";
1082         // Concatenates lines, avoid parts after // ..
1083         FOREACHC(itArg, TList, sourceCodeLines)
1084         {
1085                 const string &str=*itArg;
1086                 size_t index = str.find("//",0);
1087                 if (index == string::npos)
1088                         code += str;
1089                 else {
1090                         // We have a potential comment. Now check if it is quoted or not
1091                         bool inQuote = false;
1092                         uint i = 0;
1093                         for (;;)
1094                         {
1095                                 if ('"' == str[i])
1096                                         inQuote = !inQuote;
1097
1098                                 if ( !inQuote && ('/' == str[i]) )
1099                                 {
1100                                         ++i;
1101                                         if ('/' == str[i])
1102                                                 break;
1103
1104                                         code += '/';
1105                                 }
1106                                 code += str[i];
1107                                 ++i;
1108                                 if (str.size() == i)
1109                                         break;
1110                         }
1111                 }
1112
1113                 code+="\n "; // additional ..
1114         }
1115         code+="}";
1116         return compileCode(code, fullName);
1117 }
1118
1119 CSmartPtr<const AIVM::CByteCode> CCompiler::compileCode (const string &sourceCode, const string &fullName) const
1120 {
1121         bool debug = NLNET::IService::getInstance()->haveArg('d');
1122         CSmartPtr<const AIVM::CByteCode> byteCode = compileCodeYacc (sourceCode, fullName, debug, false);
1123
1124         if (debug)
1125         {
1126                 // Generate the old byte code
1127                 CSmartPtr<const AIVM::CByteCode> oldbyteCode = compileCodeOld (sourceCode, fullName, debug);
1128         }
1129
1130         return byteCode;
1131 }
1132
1133 void CCompiler::dumpByteCode (const string &sourceCode, const string &fullName, CSmartPtr<const AIVM::CByteCode> &byteCode,
1134                                                           const string &directory) const
1135 {
1136         // Build a valid filename
1137         string tmp = fullName;
1138         string::size_type pos;
1139         while ((pos=tmp.find (':')) != string::npos)
1140                 tmp[pos] = '-';
1141
1142         // Create the bytecode directory
1143         CFile::createDirectory (directory.c_str ());
1144
1145         // Save the bytecode
1146         nlinfo ("saving bytecode for %s", tmp.c_str ());
1147         FILE *file = fopen ((directory+"/"+tmp+".bin").c_str(), "wb");
1148         if (file)
1149         {
1150                 fwrite (&byteCode->_opcodes[0], sizeof(size_t), byteCode->_opcodes.size (), file);
1151                 fclose (file);
1152         }
1153         else
1154                 nlwarning ("can't open %s for writing", tmp.c_str ());
1155
1156         // Create the source directory
1157         CFile::createDirectory ("iasources");
1158
1159         // Save the source code
1160         file = fopen (("iasources/"+tmp+".src").c_str(), "w");
1161         if (file)
1162         {
1163                 fwrite (sourceCode.c_str (), sourceCode.size(), 1, file);
1164                 fclose (file);
1165         }
1166         else
1167                 nlstopex(("can't open %s for writing", tmp.c_str ()));
1168 }
1169
1170 CSmartPtr<const AIVM::CByteCode> CCompiler::compileCodeOld (const string &sourceCode, const string &fullName, bool debug) const
1171 {
1172         CSmartPtr<AIVM::CByteCode> byteCode = new AIVM::CByteCode(fullName);
1173
1174         nldebug("script compilation of %s", fullName.c_str());
1175         string code=sourceCode;
1176         try
1177         {
1178                 nldebug(">parsing source code ..");
1179                 CSmartPtr<CSubRuleTracer> tracer=buildCodeTree (code);
1180
1181                 if (tracer!=NULL)
1182                 {
1183                         nldebug(">generating code tree ..");
1184                         tracer=tracer->codifyTree       ();                     // removes ambiguities, at this point a good or bad code is compiled.
1185
1186                         nldebug(">generating byte code ..");
1187                         tracer->getHigherParent()->generateCode(byteCode);
1188                 }
1189                 else
1190                 {
1191                         nldebug(">empty source code ..");
1192                 }
1193
1194                 nldebug("compilation success. (code size %d)",byteCode->_opcodes.size()*4);
1195
1196                 CSmartPtr<const AIVM::CByteCode> tmp=&(*byteCode);
1197
1198                 if (debug)
1199                         dumpByteCode (sourceCode, fullName, tmp, "iaoldbytecode");
1200
1201                 return tmp;
1202         }
1203         catch (const Exception &e)
1204         {
1205                 nlwarning("compilation failed for %s", fullName.c_str());
1206                 nlwarning(e.what());
1207         }
1208
1209         return NULL;
1210 }
1211
1212 CSmartPtr<const AIVM::CByteCode> CCompiler::compileCodeYacc (const string &sourceCode, const string &fullName, bool debug, bool win32report) const
1213 {
1214         CSmartPtr<AIVM::CByteCode> byteCode = new AIVM::CByteCode(fullName);
1215
1216         if (aiCompile (byteCode->_opcodes, sourceCode.c_str (), fullName.c_str (), win32report))
1217         {
1218                 CSmartPtr<const AIVM::CByteCode> tmp=&(*byteCode);
1219
1220                 if (debug)
1221                         dumpByteCode (sourceCode, fullName, tmp, "ianewbytecode");
1222
1223                 return tmp;
1224         }
1225         else if (debug)
1226                 nlstop;
1227
1228         return NULL;
1229 }
1230
1231 void CCompiler::addToken(const string &tokenName, const string &tokenDesc)
1232 {
1233         CToken *token = new CToken(tokenName, tokenDesc);
1234         _Tokens.push_back(token);
1235 }
1236
1237 CToken *CCompiler::getToken(const  string       &tokenName)
1238 {
1239         FOREACH (tokenIt, TTokenList, _Tokens)
1240         {
1241                 if ((*tokenIt)->getName()==tokenName)
1242                         return *tokenIt;
1243         }
1244         nlassert(false);
1245         return NULL;
1246 }
1247
1248 /// Helper function
1249 static void displayErrorLinesForIndex(const string &text, size_t &index)
1250 {
1251         AIVM::CStringSeparator sep(text,"\n\r");
1252         size_t totalIndex = 0;
1253         string tmp;
1254         size_t lineIndex = 0;
1255
1256         while (sep.hasNext())
1257         {
1258                 tmp = sep.get();
1259                 const size_t stringSize = tmp.size()+1; // +1 for the \n text.
1260                 if (totalIndex+stringSize>index)
1261                         break;
1262                 totalIndex += stringSize;
1263                 lineIndex++;
1264         }
1265
1266         {
1267                 string lineoStr("at line ");
1268                 lineoStr += toString(lineIndex);
1269                 nlwarning(lineoStr.c_str());
1270         }
1271
1272         nlwarning(tmp.c_str());
1273         {
1274                 string indexerStr;
1275                 indexerStr.resize(index-totalIndex,' ');
1276                 indexerStr += "^";
1277                 nlwarning(indexerStr.c_str());
1278         }
1279 }
1280
1281
1282 bool CCompiler::getNextToken(const string &text, size_t &index, string &tokenName, string &textValue)
1283 {
1284         char c=text.at(index);
1285         while (c==' '||c=='\n'||c=='\r'||c=='\t') // to avoid blanks, returns and Tabs.
1286         {
1287                 index++;
1288                 if (index==text.size())
1289                         return false;
1290                 c = text.at(index);
1291         }
1292         const size_t firstIndex=index;
1293
1294         static const string unknownS("unknown");
1295         FOREACH(tokenIt, TTokenList, _Tokens)
1296         {
1297                 CTokenTestResult res=(*tokenIt)->buildTree(text, index);
1298                 if (res.isValid()) // we found it !!
1299                 {
1300                         tokenName = (*tokenIt)->getName();
1301                         textValue = text.substr(firstIndex, index-firstIndex);
1302                         return true;
1303                 }
1304
1305         }
1306         displayErrorLinesForIndex(text, index);
1307         throw (EScriptError("(Unrecognized pattern)", index));
1308         nlassert(false);
1309         return false;
1310 }
1311
1312 void CCompiler::addOpcode(const std::string &str,AIVM::CScriptVM::EOpcode const& op)
1313 {
1314         nlassert(_Opcodes.find(str)==_Opcodes.end());
1315         _Opcodes.insert(make_pair(str, op));
1316 }
1317
1318 const string &CCompiler::getOpcodeName(AIVM::CScriptVM::EOpcode const& op)
1319 {
1320         static string unk("---");
1321         FOREACHC(opIt, TOpcodeMap, _Opcodes)
1322         {
1323                 if (opIt->second==op)
1324                         return opIt->first;
1325         }
1326         return unk;
1327 }
1328
1329 void CCompiler::addNativeFunc(std::string const& signature, FScrptNativeFunc const& func)
1330 {
1331         TStringId strId = CStringMapper::map(signature);
1332         nlassert(_NativeFunctions.find(strId)==_NativeFunctions.end());
1333
1334         _NativeFunctions.insert(make_pair(strId, new CScriptNativeFuncParams(signature, func)));
1335 }
1336
1337 void CCompiler::addDeprecatedNativeFunc(std::string const& signature)
1338 {
1339         TStringId strId = CStringMapper::map(signature);
1340         nlassert(_NativeFunctions.find(strId)==_NativeFunctions.end());
1341
1342         _NativeFunctions.insert(make_pair(strId, new CScriptNativeFuncParams(signature, NULL)));
1343 }
1344
1345 CScriptNativeFuncParams* CCompiler::getNativeFunc(const std::string &funcName, const std::string &inparams, const std::string &outparams)
1346 {
1347         std::string signature;
1348         TStringId strId;
1349         TNativeFuncMap::iterator it;
1350         // Normal params
1351         signature = funcName + "_" + inparams + "_" + outparams;
1352         strId = CStringMapper::map(signature);
1353         it = _NativeFunctions.find(strId);
1354         if (it!=_NativeFunctions.end())
1355                 return it->second;
1356         // Variable arguments (va) as input
1357         signature = funcName + "_v_" + outparams;
1358         strId = CStringMapper::map(signature);
1359         it = _NativeFunctions.find(strId);
1360         if (it!=_NativeFunctions.end())
1361                 return it->second;
1362         // VA as output
1363         signature = funcName + "_" + inparams + "_v";
1364         strId = CStringMapper::map(signature);
1365         it = _NativeFunctions.find(strId);
1366         if (it!=_NativeFunctions.end())
1367                 return it->second;
1368         // VA as input and output
1369         signature = funcName + "_v_v";
1370         strId = CStringMapper::map(signature);
1371         it = _NativeFunctions.find(strId);
1372         if (it!=_NativeFunctions.end())
1373                 return it->second;
1374         return NULL;
1375 }
1376
1377 AIVM::CScriptVM::EOpcode CCompiler::getOpcodeAndValue(const std::string &str, std::string &value)
1378 {
1379         AIVM::CScriptVM::EOpcode opcode=AIVM::CScriptVM::INVALID_OPCODE;
1380         AIVM::CStringSeparator sep(str," \t");
1381         if (sep.hasNext())
1382         {
1383                 TOpcodeMap::iterator it=_Opcodes.find(sep.get());
1384                 if (it!=_Opcodes.end())
1385                         opcode=it->second;
1386         }
1387         if (sep.hasNext())
1388                 value=sep.get();
1389         return opcode;
1390 }
1391
1392 void CCompiler::addRule (const  string  &ruleName, const string &ruleDesc)
1393 {
1394         CRule *rule=getRule (ruleName);
1395         if (rule)
1396         {
1397                 rule->setDesc(ruleDesc);
1398                 return;
1399         }
1400         rule=new CRule(ruleName, ruleDesc);
1401         _Rules.push_back(rule);
1402 }
1403
1404 CSmartPtr<CRule> CCompiler::getRule (const  string      &ruleName)
1405 {
1406         FOREACH(ruleIt, TRuleList, _Rules)
1407         {
1408                 if ((*ruleIt)->_Name==ruleName)
1409                         return *ruleIt;
1410         }
1411         return NULL;
1412 }
1413
1414 CCompiler::TOpcodeMap                   CCompiler::_Opcodes;
1415 CCompiler::TRuleList                    CCompiler::_Rules;
1416 CCompiler::TTokenList                   CCompiler::_Tokens;
1417 CCompiler::TNativeFuncMap               CCompiler::_NativeFunctions;
1418 CCompiler*                                              CCompiler::_Instance = NULL;
1419 CBasicToken::TBasicTokenList    CBasicToken::_BasicTokens;
1420
1421 //////////////////////////////////////////////////////////////////////////////
1422 // CJumpRememberer implementation                                           //
1423 //////////////////////////////////////////////////////////////////////////////
1424
1425 CJumpRememberer::CJumpRememberer(size_t codeBlockIndex)
1426 : _where(~0)
1427 , _codeBlockIndex(codeBlockIndex)
1428 {
1429 }
1430
1431 //////////////////////////////////////////////////////////////////////////////
1432 // CJumpTable implementation                                                //
1433 //////////////////////////////////////////////////////////////////////////////
1434
1435 CJumpTable::CJumpTable(const CSmartPtr<AIVM::CByteCode> &byteCode)
1436 : _byteCode(byteCode)
1437 {
1438         _codeOffsets.push_back(0);
1439 }
1440
1441 CJumpTable::~CJumpTable()
1442 {
1443         newCodeBlock();
1444
1445         FOREACHC(jumpIt, vector<CJumpRememberer>, _jumps)
1446         {
1447                 nlassert(jumpIt->_where<_byteCode->_opcodes.size());
1448                 nlassert(jumpIt->_codeBlockIndex<_codeOffsets.size());
1449                 _byteCode->_opcodes[jumpIt->_where]=_codeOffsets[jumpIt->_codeBlockIndex]-jumpIt->_where;
1450         }
1451 }
1452
1453 void CJumpTable::add(CJumpRememberer jump)
1454 {
1455         jump._where=_byteCode->_opcodes.size();
1456         _jumps.push_back(jump);
1457 }
1458
1459 void CJumpTable::newCodeBlock()
1460 {
1461         _codeOffsets.push_back(_byteCode->_opcodes.size());
1462 }
1463
1464 //////////////////////////////////////////////////////////////////////////////
1465 // CCaseTracer implementation                                               //
1466 //////////////////////////////////////////////////////////////////////////////
1467
1468 CCaseTracer::CCaseTracer(const CSmartPtr<CSubRuleTracer> &tracer, const string &sourceName)
1469 : _tracer(tracer)
1470 {
1471         const CSubRuleTracer *chldTracer=tracer->getChildForName(s_kw_readConstVar);
1472         const CSubRuleTracer *valChldTracer=NULL;
1473
1474         breakable
1475         {
1476                 if (valChldTracer=chldTracer->getChildForName(s_kw_CHAIN))
1477                 {
1478                         const string &strRef=valChldTracer->_TextValue;
1479                         TStringId strId;
1480                         if ( strRef.at(0)=='"'
1481                                 && strRef.at(0)==strRef.at(strRef.size()-1))
1482                                 strId=CStringMapper::map(strRef.substr(1,strRef.size()-2));
1483                         else
1484                                 strId=CStringMapper::map(strRef);
1485                         _sortValue=*((size_t*)&strId);
1486                         break;
1487                 }
1488                 if (valChldTracer=chldTracer->getChildForName(s_kw_NUMBER))
1489                 {
1490                         const string &strRef=valChldTracer->_TextValue;
1491                         float f;
1492                         NLMISC::fromString(strRef, f);
1493                         _sortValue=*((size_t*)&f);
1494                         break;
1495                 }
1496                 if (!valChldTracer)
1497                         throw Exception("Invalid case parameter");
1498         }
1499
1500         chldTracer=tracer->getChildForName(s_kw_lineOrClose);
1501
1502         CSmartPtr<AIVM::CByteCode> bc=new AIVM::CByteCode(sourceName);
1503         chldTracer->generateCode(bc);
1504         _code=&(*bc);
1505 }
1506
1507 typedef std::map<size_t, CSmartPtr<CCaseTracer> > TCaseTracerList;
1508
1509 //////////////////////////////////////////////////////////////////////////////
1510 // CSubRuleTracer implementation                                            //
1511 //////////////////////////////////////////////////////////////////////////////
1512
1513 CSubRuleTracer::CSubRuleTracer(size_t tokenStartIndex, size_t currentTokenIndex, const string &name, const string &textValue)
1514 : _index(0)
1515 , _tokenStartIndex(tokenStartIndex)
1516 , _tokenIndex(currentTokenIndex)
1517 , _Name(name)
1518 , _TextValue(textValue)
1519 , _Valid(false)
1520 {
1521         updatePreviousNext();
1522 }
1523
1524 CSubRuleTracer::CSubRuleTracer(NLMISC::CSmartPtr<CSubRule> subRule, size_t tokenStartIndex, size_t currentTokenIndex, const std::string &name, const std::string &textValue)
1525 : _index(0)
1526 , _tokenStartIndex(tokenStartIndex)
1527 , _tokenIndex(currentTokenIndex)
1528 , _Name(name)
1529 , _TextValue(textValue)
1530 , _Valid(false)
1531 , _subRule(subRule)
1532 {
1533         updatePreviousNext();
1534 }
1535
1536 CSubRuleTracer::CSubRuleTracer(const CSubRuleTracer &otherSRT)
1537 : _index(otherSRT._index)
1538 , _tokenStartIndex(otherSRT._tokenStartIndex)
1539 , _tokenIndex(otherSRT._tokenIndex)
1540 , _Name(otherSRT._Name)
1541 , _TextValue(otherSRT._TextValue)
1542 , _Valid(otherSRT._Valid)
1543 , _subRule(otherSRT._subRule)
1544 {
1545         updatePreviousNext();
1546 }
1547
1548 CSubRuleTracer::~CSubRuleTracer()
1549 {
1550 }
1551
1552 void CSubRuleTracer::updatePreviousNext()
1553 {
1554         // Next registration.
1555         {
1556                 CSubRuleTracer::TOrderedTracers::iterator it=CSubRuleTracer::_NextTracers.find(_tokenStartIndex);
1557                 if (it==CSubRuleTracer::_NextTracers.end())
1558                 {
1559                         CSubRuleTracer::_NextTracers.insert(make_pair(_tokenStartIndex, CSubRuleTracer::TTracersSet()) );
1560                         it=CSubRuleTracer::_NextTracers.find(_tokenStartIndex);
1561                         nlassert(it!=CSubRuleTracer::_NextTracers.end());
1562                 }
1563                 CSubRuleTracer::TTracersSet &set=it->second;
1564                 set.insert(this);
1565         }
1566
1567         // Previous registration.
1568         {
1569                 CSubRuleTracer::TOrderedTracers::iterator it=CSubRuleTracer::_PreviousTracers.find(_tokenIndex);
1570                 if (it==CSubRuleTracer::_PreviousTracers.end())
1571                 {
1572                         CSubRuleTracer::_PreviousTracers.insert(make_pair(_tokenIndex, CSubRuleTracer::TTracersSet()) );
1573                         it=CSubRuleTracer::_PreviousTracers.find(_tokenIndex);
1574                         nlassert(it!=CSubRuleTracer::_PreviousTracers.end());
1575                 }
1576                 CSubRuleTracer::TTracersSet &set=it->second;
1577                 set.insert(this);
1578         }
1579 }
1580
1581 CSubRuleTracer::TOrderedTracers CSubRuleTracer::_PreviousTracers;
1582 CSubRuleTracer::TOrderedTracers CSubRuleTracer::_NextTracers;
1583
1584 // Removers ------------------------------------------------------------------
1585
1586 void CSubRuleTracer::removeParent(CSubRuleTracer *tracer)
1587 {
1588         FOREACH(itParent, TSubRuleTracerList, _parentTracers)
1589         {
1590                 if ((*itParent)==tracer)
1591                 {
1592                         _parentTracers.erase(itParent);
1593                         return;
1594                 }
1595         }
1596         nlassert(false);
1597 }
1598 void CSubRuleTracer::removeChild(CSubRuleTracer *tracer)
1599 {
1600         FOREACH(itChild, TSubRuleTracerList, _childTracers)
1601         {
1602                 if ((*itChild)==tracer)
1603                 {
1604                         _childTracers.erase(itChild);
1605                         return;
1606                 }
1607         }
1608         nlassert(false);
1609 }
1610
1611 // ---------------------------------------------------------------------------
1612
1613 void CSubRuleTracer::detachFromEveryBody()
1614 {
1615         FOREACH(it, TSubRuleTracerList, _parentTracers)
1616                 (*it)->removeChild(this);
1617         FOREACH(it, TSubRuleTracerList, _childTracers)
1618                 (*it)->removeParent(this);
1619 }
1620
1621 void CSubRuleTracer::iterateToMarkValidTracer()
1622 {
1623         _Valid=true;
1624         FOREACH(childIt, TSubRuleTracerList, _childTracers)
1625                 (*childIt)->iterateToMarkValidTracer();
1626 }
1627
1628 CSmartPtr<CSubRuleTracer> CSubRuleTracer::getValidTracer() const
1629 {
1630         CSmartPtr<CSubRuleTracer> sRT=new       CSubRuleTracer(*this);
1631
1632         sRT->_childTracers.reserve(_childTracers.size());
1633         FOREACHC(childIt, TSubRuleTracerList, _childTracers)
1634         {
1635                 CSmartPtr<CSubRuleTracer> child=(*childIt)->getValidTracer();
1636                 child->_parentTracers.push_back(sRT);
1637                 sRT->_childTracers.push_back(child);
1638         }
1639
1640         return sRT;
1641 }
1642
1643 void CSubRuleTracer::flushErrors        ()
1644 {
1645         FOREACH(itChild, TSubRuleTracerList, _childTracers)
1646         {
1647                 nlassert((*itChild)->_parentTracers.size()==1); // if there is a problem, we can see it here .. :)
1648                 (*itChild)->flushErrors();
1649         }
1650 }
1651
1652 void CSubRuleTracer::removeInvalidTracers()
1653 {
1654         TSubRuleTracerList parentList=_parentTracers; // copy to avoid problems.
1655
1656         if ( !_Valid
1657                 && _childTracers.size()>0) // if not valid and not a base tracer.
1658         {
1659                 detachFromEveryBody();
1660         }
1661
1662         FOREACH(parentIt, TSubRuleTracerList, parentList)
1663                 (*parentIt)->removeInvalidTracers();
1664 }
1665
1666
1667 CSmartPtr<CSubRuleTracer> CSubRuleTracer::codifyTree()
1668 {
1669         if (getHigherParent()==this) // an error occurred.
1670         {
1671                 bool errorAppened=false;
1672                 // Check.
1673                 {
1674                         CSubRuleTracer *tracer=this;
1675                         while (tracer!=NULL)
1676                         {
1677                                 if (tracer->_parentTracers.size()==0) // if there is a problem, we can see it here .. :)
1678                                 {
1679                                         errorAppened=true;
1680                                         nlwarning("an grammar error appeared that breaks this enclosing: ");
1681                                         tracer->dump(10);
1682                                 }
1683                                 tracer=tracer->getNextLower();
1684                         }
1685                 }
1686                 // Flush errors.
1687                 getHigherParent()->flushErrors();
1688                 if (errorAppened)
1689                         throw Exception("Script Grammar Error");
1690                 return getHigherParent();
1691         }
1692         else
1693         {
1694                 CSmartPtr<CSubRuleTracer> returnTracer=getHigherParent()->getValidTracer();
1695
1696                 // Flush errors.
1697                 returnTracer->flushErrors();
1698                 return returnTracer;
1699         }
1700
1701 }
1702
1703 CSubRuleTracer *CSubRuleTracer::getNextLower() const
1704 {
1705         CSubRuleTracer::TOrderedTracers::iterator it=CSubRuleTracer::_NextTracers.find(_tokenIndex+1);
1706         if (it!=CSubRuleTracer::_NextTracers.end())
1707         {
1708                 CSubRuleTracer::TTracersSet &set=it->second;
1709                 FOREACH(setIt, CSubRuleTracer::TTracersSet, set)
1710                 {
1711                         if ((*setIt)->_childTracers.size()==0)
1712                                 return (*setIt);
1713                 }
1714         }
1715         return NULL;
1716 }
1717
1718
1719 CSubRuleTracer *CSubRuleTracer::getHigherParent()
1720 {
1721         if (_parentTracers.size()>0)
1722                 return _parentTracers.back()->getHigherParent();
1723         return this;
1724 }
1725
1726 // Rule finding can be optimized here with an multimap tree.
1727 void CSubRuleTracer::checkRules(size_t currentToken)
1728 {
1729         FOREACH(ruleIt, CCompiler::TRuleList, CCompiler::_Rules )
1730         {
1731                 FOREACH(subRuleIt, TSubRuleList, (*ruleIt)->_subRules)
1732                 {
1733                         TSubRuleTracerList childTracers;
1734                         checkRule(*subRuleIt,0, currentToken, childTracers);
1735                 }
1736         }
1737 }
1738
1739 void CSubRuleTracer::checkRule(CSubRule *rule, size_t index, size_t currentToken, TSubRuleTracerList &childTracers)
1740 {
1741         bool equal=false;
1742         string token=rule->_tokens[rule->_tokens.size()-1-index];
1743         if (token.at(0)=='+')
1744                 equal=(_Name==token.substr(1,token.size()-1));
1745         else
1746                 equal=(_Name==token);
1747
1748         if (!equal) // if not equal, check if its equal to the previous one. (and if this one was 'multi')
1749         {
1750                 if (index==0) // failed.
1751                         return;
1752
1753                 token=rule->_tokens[rule->_tokens.size()-index];
1754                 if (token.at(0)!='+')
1755                         return;
1756
1757                 equal=(_Name==token.substr(1,token.size()-1));
1758                 if (!equal) // failed.
1759                         return;
1760                 index--;
1761         }
1762
1763         childTracers.push_back(this);
1764
1765         if (index==rule->_tokens.size()-1) // do we match a rule here ?
1766         {
1767                 CSubRuleTracer *lastTracer=childTracers.back();
1768
1769                 CSmartPtr<CSubRuleTracer> newTracer=new CSubRuleTracer(rule, lastTracer->_tokenStartIndex, currentToken, rule->_Parent->_Name, "");
1770
1771                 // we have to copy this vector inverted ..
1772                 newTracer->_childTracers.resize(childTracers.size());
1773                 std::copy(childTracers.begin(), childTracers.end(), newTracer->_childTracers.rbegin());
1774
1775                 FOREACH(childIt, TSubRuleTracerList, childTracers)
1776                         (*childIt)->_parentTracers.push_back(newTracer);
1777
1778                 // Recursively trying to complete others higher rules starting with this new Tracer. :)
1779                 newTracer->checkRules(currentToken);
1780         }
1781         else
1782         {
1783                 // _previousTracers cannot be affected by checkRule calls.
1784
1785                 if ((_tokenStartIndex-1)>=0)
1786                 {
1787                         CSubRuleTracer::TOrderedTracers::iterator it=CSubRuleTracer::_PreviousTracers.find(_tokenStartIndex-1);
1788                         if (it!=CSubRuleTracer::_PreviousTracers.end())
1789                         {
1790                                 CSubRuleTracer::TTracersSet &set=it->second;
1791                                 FOREACH(setIt, CSubRuleTracer::TTracersSet, set)
1792                                         (*setIt)->checkRule(rule, index+1, currentToken, childTracers);
1793                         }
1794                 }
1795         }
1796         nlassert(childTracers.back()==(CSubRuleTracer*)this);
1797         childTracers.pop_back();
1798 }
1799
1800 void CSubRuleTracer::dump(size_t indent) const
1801 {
1802         string str;
1803         str.resize(indent,' ');
1804         str+=_Name;
1805         str+="("+toString(_tokenStartIndex);
1806         str+=","+toString(_tokenIndex);
1807         str+="): "+_TextValue;
1808
1809         nldebug(str.c_str());
1810         FOREACHC(itTracer, TSubRuleTracerList, _childTracers)
1811                 (*itTracer)->dump(indent+1);
1812 }
1813
1814 const CSubRuleTracer *CSubRuleTracer::getChildForName(const string &name) const
1815 {
1816         FOREACHC(childIt, TSubRuleTracerList, _childTracers)
1817         {
1818                 if ((*childIt)->_Name==name)
1819                         return *childIt;
1820         }
1821         return NULL;
1822 }
1823
1824 size_t CSubRuleTracer::getNbChildNamed(const string &name) const
1825 {
1826         size_t nb=0;
1827         if (_Name==name)
1828                 nb=1;
1829
1830         FOREACHC(childIt, TSubRuleTracerList, _childTracers)
1831                 nb+=(*childIt)->getNbChildNamed(name);
1832         return nb;
1833 }
1834
1835 void CSubRuleTracer::getSignature(string &signature, bool inOtherWiseOut) const
1836 {
1837         if (inOtherWiseOut)
1838         {
1839                 if ( _Name==s_kw_exp
1840                         || _Name==s_kw_somme
1841                         || _Name==s_kw_produit
1842                         || _Name==s_kw_facteur
1843                         || _Name==s_kw_NAME
1844                         || _Name==s_kw_NUMBER)
1845                 {
1846                         signature+="f";
1847                         return;
1848                 }
1849                 if ( _Name==s_kw_CHAIN
1850                         || _Name==s_kw_STRNAME)
1851                 {
1852                         signature+="s";
1853                         return;
1854                 }
1855                 if ( _Name==s_kw_POINT)
1856                 {
1857                         signature.resize(signature.size()-1);
1858                         return;
1859                 }
1860         }
1861         else
1862         {
1863                 if (_Name==s_kw_NAME)
1864                 {
1865                         signature+="f";
1866                         return;
1867                 }
1868                 if (_Name==s_kw_STRNAME)
1869                 {
1870                         signature+="s";
1871                         return;
1872                 }
1873                 if ( _Name==s_kw_POINT)
1874                 {
1875                         signature.resize(signature.size()-1);
1876                         return;
1877                 }
1878                 if ( _Name==s_kw_exp
1879                         || _Name==s_kw_somme
1880                         || _Name==s_kw_produit
1881                         || _Name==s_kw_facteur
1882                         || _Name==s_kw_CHAIN
1883                         || _Name==s_kw_NUMBER)
1884                 {
1885                         signature+="!";
1886                         return;
1887                 }
1888         }
1889         FOREACHC(childIt, TSubRuleTracerList, _childTracers)
1890                 (*childIt)->getSignature(signature,inOtherWiseOut);
1891 }
1892
1893 void CSubRuleTracer::generateCode(CSmartPtr<AIVM::CByteCode> &cByteCode) const
1894 {
1895         using namespace AIVM;
1896
1897         nlassert(!cByteCode.isNull());
1898         typedef vector<CSmartPtr<CByteCode> > TCodePieceList;
1899         size_t          codeBlockIndex=0;
1900         TCodePieceList codePieces;
1901         vector<size_t> &byteCode=cByteCode->_opcodes;
1902         CJumpTable      jumpTable(cByteCode);
1903
1904         size_t randCountMarkIndex=~0;
1905
1906         {
1907                 codePieces.resize(_childTracers.size());
1908
1909                 TCodePieceList::iterator itPiece=codePieces.begin();
1910                 FOREACHC(childIt, TSubRuleTracerList, _childTracers)
1911                 {
1912                         *itPiece=new CByteCode(cByteCode->_sourceName);
1913                         (*childIt)->generateCode(*itPiece);
1914                         ++itPiece;
1915                 }
1916
1917         }
1918
1919         if (!_subRule.isNull())
1920         {
1921
1922                 FOREACHC(instrIt, vector<string>, _subRule->_ExecOpCodes)
1923                 {
1924                         const string str=*instrIt;
1925                         string param;
1926                         const CScriptVM::EOpcode op=CCompiler::getOpcodeAndValue(str, param);
1927
1928                         if (op!=CScriptVM::INVALID_OPCODE) // it could something else than an instruction.
1929                         {
1930                                 switch(op)
1931                                 {
1932                                 case CScriptVM::JE:
1933                                 case CScriptVM::JNE:
1934                                 case CScriptVM::JUMP:
1935                                         byteCode.push_back(op); // + Jump offset.
1936
1937                                         uint32 index;
1938                                         NLMISC::fromString(param, index);
1939                                         jumpTable.add(CJumpRememberer(index));
1940                                         byteCode.push_back(0); // Invalid
1941                                         break;
1942                                 default:
1943                                         byteCode.push_back(op);
1944                                         break;
1945                                 }
1946                                 jumpTable.newCodeBlock();
1947                                 continue;
1948                         }
1949
1950                         breakable
1951                         {
1952                                 if (str.find("Atof")!=string::npos)
1953                                 {
1954                                         uint32 index;
1955                                         NLMISC::fromString(param, index);
1956                                         --index;
1957                                         string &strRef=_childTracers[index]->_TextValue;
1958                                         float f;
1959                                         NLMISC::fromString(strRef, f);
1960                                         byteCode.push_back(*((size_t*)&f));
1961                                         jumpTable.newCodeBlock();
1962                                         break;
1963                                 }
1964
1965                                 if (str.find("String")!=string::npos)
1966                                 {
1967                                         uint32 index;
1968                                         NLMISC::fromString(param, index);
1969                                         --index;
1970                                         string &strRef=_childTracers[index]->_TextValue;
1971                                         TStringId strId;
1972                                         if ( strRef.at(0)=='"'
1973                                                 && strRef.at(0)==strRef.at(strRef.size()-1))
1974                                                 strId=CStringMapper::map(strRef.substr(1,strRef.size()-2));
1975                                         else
1976                                                 strId=CStringMapper::map(strRef);
1977                                         byteCode.push_back(*((size_t*)&strId));
1978                                         jumpTable.newCodeBlock();
1979                                         break;
1980                                 }
1981
1982                                 if (str.find("CodeAllExceptFirstAndLast")!=string::npos)
1983                                 {
1984                                         size_t index=0;
1985
1986                                         FOREACHC(CPIt, TCodePieceList, codePieces)
1987                                         {
1988                                                 index++;                                        // Not the first, not the last.
1989                                                 if ( index==1
1990                                                         || index==codePieces.size())
1991                                                         continue;
1992
1993                                                 if (byteCode.size()==0)
1994                                                         byteCode=(*CPIt)->_opcodes;
1995                                                 else
1996                                                 {
1997                                                         FOREACHC(codePieceIt, vector<size_t>, (*CPIt)->_opcodes)
1998                                                                 byteCode.push_back(*codePieceIt);
1999                                                 }
2000                                         }
2001
2002                                         jumpTable.newCodeBlock();
2003                                         break;
2004                                 }
2005
2006                                 if (str.find("AllCode")!=string::npos)
2007                                 {
2008
2009                                         FOREACHC(CPIt, TCodePieceList, codePieces)
2010                                         {
2011                                                 if (byteCode.size()==0)
2012                                                         byteCode=(*CPIt)->_opcodes;
2013                                                 else
2014                                                 {
2015                                                         FOREACHC(codePieceIt, vector<size_t>, (*CPIt)->_opcodes)
2016                                                                 byteCode.push_back(*codePieceIt);
2017                                                 }
2018                                         }
2019
2020                                         jumpTable.newCodeBlock();
2021                                         break;
2022                                 }
2023
2024                                 if (str.find("Code")!=string::npos)
2025                                 {
2026                                         uint32 index;
2027                                         NLMISC::fromString(param, index);
2028                                         --index;
2029                                         if (byteCode.size()==0)
2030                                                 byteCode=codePieces[index]->_opcodes;
2031                                         else
2032                                         {
2033                                                 FOREACHC(codePieceIt, vector<size_t>, codePieces[index]->_opcodes)
2034                                                         byteCode.push_back(*codePieceIt);
2035                                         }
2036                                         jumpTable.newCodeBlock();
2037                                         break;
2038                                 }
2039
2040                                 if (str.find("NativeCall")!=string::npos)
2041                                 {
2042                                         string funcName;
2043                                         string inParamsSig;
2044                                         string outParamsSig;
2045                                         // Extract signature
2046                                         {
2047                                                 const CSubRuleTracer*paramTracer=getChildForName(s_kw_NAME);
2048                                                 funcName=paramTracer->_TextValue;
2049
2050                                                 paramTracer=getChildForName(s_kw_params);
2051                                                 if (!paramTracer)
2052                                                         throw Exception("right params not found for the native call "+paramTracer->_TextValue);
2053                                                 paramTracer->getSignature(inParamsSig, true);
2054
2055                                                 paramTracer=getChildForName(s_kw_tuple);
2056                                                 if (!paramTracer)
2057                                                         throw Exception("left params(tuple) not found for the native call "+paramTracer->_TextValue);
2058                                                 paramTracer->getSignature(outParamsSig, false);
2059                                         }
2060                                         // Get a function description
2061                                         CScriptNativeFuncParams *funcParam=CCompiler::getNativeFunc(funcName, inParamsSig, outParamsSig);
2062                                         if (!funcParam)
2063                                         {
2064                                                 string signature = funcName + "_" + inParamsSig + "_" + outParamsSig;
2065                                                 throw Exception("Critical: unknown function name or bad parameters "+signature);
2066                                         }
2067
2068                                         size_t mode = 0;
2069                                         if (funcParam->_va)
2070                                                 mode |= 1; // :KLUDGE: Hardcoded 1 :TODO: replace with a named constant
2071
2072                                         byteCode.push_back(CScriptVM::NATIVE_CALL);
2073                                 //      byteCode.push_back(*((size_t*)&funcParam));
2074                                         TStringId strId;
2075                                         strId = CStringMapper::map(funcName);
2076                                         byteCode.push_back(*((size_t*)&strId));
2077                                         byteCode.push_back(mode);
2078                                         strId = CStringMapper::map(inParamsSig);
2079                                         byteCode.push_back(*((size_t*)&strId));
2080                                         strId = CStringMapper::map(outParamsSig);
2081                                         byteCode.push_back(*((size_t*)&strId));
2082
2083                                         jumpTable.newCodeBlock();
2084                                         break;
2085                                 }
2086
2087                                 if (str.find("RandomSeq")!=string::npos)
2088                                 {
2089                                         const CSubRuleTracer *randomTracers=getChildForName(s_kw_expeclose);
2090                                         const size_t nbTracers=randomTracers->_childTracers.size()-2; // LA and RA are omitted.
2091
2092                                         byteCode.push_back(CScriptVM::RAND);
2093                                         byteCode.push_back(nbTracers);
2094
2095                                         byteCode.push_back(CScriptVM::JUMP);
2096                                         jumpTable.add(CJumpRememberer(nbTracers+1)); // Final Jump Position.
2097                                         byteCode.push_back(0); // Invalid
2098
2099                                         size_t tracerInd=nbTracers;
2100                                         while (tracerInd>0)
2101                                         {
2102                                                 byteCode.push_back(CScriptVM::JUMP);
2103                                                 jumpTable.add(CJumpRememberer(tracerInd));
2104                                                 byteCode.push_back(0); // Invalid
2105                                                 tracerInd--;
2106                                         }
2107                                         byteCode.push_back(CScriptVM::RANDEND);
2108                                         jumpTable.newCodeBlock();
2109
2110                                         {
2111                                                 codePieces.resize(0);
2112                                                 codePieces.resize(randomTracers->_childTracers.size());
2113
2114                                                 TCodePieceList::iterator itPiece=codePieces.begin();
2115                                                 FOREACHC(childIt, TSubRuleTracerList, randomTracers->_childTracers)
2116                                                 {
2117                                                         *itPiece=new AIVM::CByteCode(cByteCode->_sourceName);
2118                                                         (*childIt)->generateCode(*itPiece);
2119                                                         ++itPiece;
2120                                                 }
2121                                         }
2122
2123                                         tracerInd=nbTracers;
2124                                         while (tracerInd>0)
2125                                         {
2126                                                 tracerInd--;
2127                                                 if (byteCode.size()==0)
2128                                                         byteCode=codePieces[tracerInd+1]->_opcodes;
2129                                                 else
2130                                                 {
2131                                                         FOREACHC(codePieceIt, vector<size_t>, codePieces[tracerInd+1]->_opcodes)
2132                                                                 byteCode.push_back(*codePieceIt);
2133                                                 }
2134                                                 byteCode.push_back(CScriptVM::RET);
2135                                                 jumpTable.newCodeBlock();
2136                                         }
2137                                         break;
2138                                 }
2139
2140                                 if (str.find("SwitchSeq")!=string::npos)
2141                                 {
2142                                         // first, build a list of case statements.
2143                                         TCaseTracerList caseTracers;
2144                                         FOREACHC(chldIt, TSubRuleTracerList, _childTracers)
2145                                         {
2146                                                 if ((*chldIt)->_Name==s_kw_case)
2147                                                 {
2148                                                         CSmartPtr<CCaseTracer> ptr=new CCaseTracer(*chldIt, cByteCode->_sourceName);
2149                                                         caseTracers.insert(make_pair(ptr->_sortValue, ptr));
2150                                                 }
2151                                         }
2152
2153                                         byteCode.push_back(CScriptVM::SWITCH);
2154                                         byteCode.push_back(caseTracers.size());
2155
2156                                         jumpTable.add(CJumpRememberer(caseTracers.size()+2)); // Final Jump Position.
2157                                         byteCode.push_back(0); // Invalid
2158
2159                                         {
2160                                                 size_t index=2;
2161                                                 FOREACHC(caseIt, TCaseTracerList, caseTracers)
2162                                                 {
2163                                                         byteCode.push_back(caseIt->first);
2164                                                         jumpTable.add(CJumpRememberer(index)); // Final Jump Position.
2165                                                         byteCode.push_back(0); // Invalid
2166                                                         index++;
2167                                                 }
2168                                         }
2169                                         jumpTable.newCodeBlock();
2170
2171                                         FOREACHC(caseIt, TCaseTracerList, caseTracers)
2172                                         {
2173                                                 FOREACHC(codePieceIt, vector<size_t>, caseIt->second->_code->_opcodes)
2174                                                         byteCode.push_back(*codePieceIt);
2175                                                 byteCode.push_back(CScriptVM::RET);
2176                                                 jumpTable.newCodeBlock();
2177                                         }
2178                                         break;
2179                                 }
2180                                 throw Exception("Unrecognized keyword "+str);
2181                         }
2182                 }
2183         }
2184 }
2185
2186 /*
2187         if (outputFilename != "") output the byte code in a file
2188 */
2189 bool compileExternalScript (const char *filename, const char *outputFilename)
2190 {
2191         // Read the content
2192         bool result = false;
2193         FILE *file = fopen (filename, "r");
2194         if (file)
2195         {
2196                 string content;
2197                 char buffer[512];
2198                 int read;
2199                 while ((read = (int)fread (buffer, 1, sizeof(buffer)-1, file)) == sizeof(buffer)-1)
2200                 {
2201                         buffer[read] = 0;
2202                         content += buffer;
2203                 }
2204                 buffer[read] = 0;
2205                 content += buffer;
2206                 CSmartPtr<const AIVM::CByteCode> byteCode = CCompiler::getInstance ().compileCodeYacc (content, filename, NLNET::IService::getInstance()->haveArg('d'), true);
2207                 fclose (file);
2208                 if (byteCode)
2209                 {
2210                         // Save the byte code ?
2211                         if (strcmp (outputFilename, "") != 0)
2212                         {
2213                                 FILE *output = fopen (filename, "wb");
2214                                 if (output)
2215                                 {
2216                                         size_t size = byteCode->_opcodes.size()*sizeof(size_t);
2217                                         if (fwrite (&byteCode->_opcodes[0], 1, size, output) == size)
2218                                                 result = true;
2219                                         else
2220                                                 nlwarning ("Error while writing %s", outputFilename);
2221                                         fclose (output);
2222                                 }
2223                                 else
2224                                         nlwarning ("Can't open the file %s for writing", outputFilename);
2225                         }
2226                         else
2227                                 result = true;
2228                 }
2229         }
2230         else
2231                 nlwarning ("Can't open the file %s for reading", filename);
2232         return result;
2233 }
2234
2235 //////////////////////////////////////////////////////////////////////////////
2236 /*
2237 */
2238
2239 }; // namespace
2240
2241 NLMISC_COMMAND(listNativeFunctions, "list native functions of that AIS", "")
2242 {
2243         CLogStringWriter stringWriter(&log);
2244         AICOMP::CCompiler::TNativeFuncMap const& funcs = AICOMP::CCompiler::getInstance().getFunctionList();
2245         std::deque<std::string> names;
2246         FOREACHC(itFunc, AICOMP::CCompiler::TNativeFuncMap, funcs)
2247                 names.push_back(CStringMapper::unmap(itFunc->first));
2248         std::sort(names.begin(), names.end());
2249         FOREACHC(itName, std::deque<std::string>, names)
2250                 log.displayNL("%s", itName->c_str());
2251         return true;
2252 }