thirdparty/libantlr3c-3.2/src/antlr3tokenstream.c.orig

   1 /// \file
   2 /// Default implementation of CommonTokenStream
   3 ///
   4
   5 // [The "BSD licence"]
   6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
   7 // http://www.temporal-wave.com
   8 // http://www.linkedin.com/in/jimidle
   9 //
  10 // All rights reserved.
  11 //
  12 // Redistribution and use in source and binary forms, with or without
  13 // modification, are permitted provided that the following conditions
  14 // are met:
  15 // 1. Redistributions of source code must retain the above copyright
  16 //    notice, this list of conditions and the following disclaimer.
  17 // 2. Redistributions in binary form must reproduce the above copyright
  18 //    notice, this list of conditions and the following disclaimer in the
  19 //    documentation and/or other materials provided with the distribution.
  20 // 3. The name of the author may not be used to endorse or promote products
  21 //    derived from this software without specific prior written permission.
  22 //
  23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33
  34 #include    <antlr3tokenstream.h>
  35
  36 #ifdef  ANTLR3_WINDOWS
  37 #pragma warning( disable : 4100 )
  38 #endif
  39
  40 // COMMON_TOKEN_STREAM API
  41 //
  42 static void                                     setTokenTypeChannel     (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
  43 static void                                     discardTokenType        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
  44 static void                                     discardOffChannel       (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
  45 static pANTLR3_VECTOR           getTokens                       (pANTLR3_COMMON_TOKEN_STREAM cts);
  46 static pANTLR3_LIST                     getTokenRange           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
  47 static pANTLR3_LIST                     getTokensSet            (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
  48 static pANTLR3_LIST                     getTokensList           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
  49 static pANTLR3_LIST                     getTokensType           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
  50
  51 // TOKEN_STREAM API
  52 //
  53 static pANTLR3_COMMON_TOKEN tokLT                               (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
  54 static pANTLR3_COMMON_TOKEN dbgTokLT                    (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
  55 static pANTLR3_COMMON_TOKEN get                                 (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
  56 static pANTLR3_TOKEN_SOURCE getTokenSource              (pANTLR3_TOKEN_STREAM ts);
  57 static void                                     setTokenSource          (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
  58 static pANTLR3_STRING       toString                    (pANTLR3_TOKEN_STREAM ts);
  59 static pANTLR3_STRING       toStringSS                  (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
  60 static pANTLR3_STRING       toStringTT                  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
  61 static void                                     setDebugListener        (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
  62
  63 // INT STREAM API
  64 //
  65 static void                                     consume                                         (pANTLR3_INT_STREAM is);
  66 static void                                     dbgConsume                                      (pANTLR3_INT_STREAM is);
  67 static ANTLR3_UINT32        _LA                                                 (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
  68 static ANTLR3_UINT32        dbgLA                                               (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
  69 static ANTLR3_MARKER        mark                                                (pANTLR3_INT_STREAM is);
  70 static ANTLR3_MARKER        dbgMark                                             (pANTLR3_INT_STREAM is);
  71 static void                                     release                                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
  72 static ANTLR3_UINT32        size                                                (pANTLR3_INT_STREAM is);
  73 static ANTLR3_MARKER            tindex                                          (pANTLR3_INT_STREAM is);
  74 static void                                     rewindStream                            (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
  75 static void                                     dbgRewindStream                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
  76 static void                                     rewindLast                                      (pANTLR3_INT_STREAM is);
  77 static void                                     dbgRewindLast                           (pANTLR3_INT_STREAM is);
  78 static void                                     seek                                            (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
  79 static void                                     dbgSeek                                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
  80 static pANTLR3_STRING           getSourceName                           (pANTLR3_INT_STREAM is);
  81 static void                                     antlr3TokenStreamFree           (pANTLR3_TOKEN_STREAM       stream);
  82 static void                                     antlr3CTSFree                           (pANTLR3_COMMON_TOKEN_STREAM    stream);
  83
  84 // Helpers
  85 //
  86 static void                                     fillBuffer                                      (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
  87 static ANTLR3_UINT32        skipOffTokenChannels                (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
  88 static ANTLR3_UINT32        skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
  89 static pANTLR3_COMMON_TOKEN LB                                                  (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
  90
  91 ANTLR3_API pANTLR3_TOKEN_STREAM
  92 antlr3TokenStreamNew()
  93 {
  94     pANTLR3_TOKEN_STREAM stream;
  95
  96     // Memory for the interface structure
  97     //
  98     stream  = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
  99
 100     if  (stream == NULL)
 101     {
 102                 return  NULL;
 103     }
 104
 105     // Install basic API
 106     //
 107     stream->free    =  antlr3TokenStreamFree;
 108
 109
 110     return stream;
 111 }
 112
 113 static void
 114 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
 115 {
 116     ANTLR3_FREE(stream);
 117 }
 118
 119 static void
 120 antlr3CTSFree       (pANTLR3_COMMON_TOKEN_STREAM stream)
 121 {
 122         // We only free up our subordinate interfaces if they belong
 123         // to us, otherwise we let whoever owns them deal with them.
 124         //
 125         if      (stream->tstream->super == stream)
 126         {
 127                 if      (stream->tstream->istream->super == stream->tstream)
 128                 {
 129                         stream->tstream->istream->free(stream->tstream->istream);
 130                         stream->tstream->istream = NULL;
 131                 }
 132                 stream->tstream->free(stream->tstream);
 133         }
 134
 135         // Now we free our own resources
 136         //
 137         if      (stream->tokens != NULL)
 138         {
 139                 stream->tokens->free(stream->tokens);
 140                 stream->tokens  = NULL;
 141         }
 142         if      (stream->discardSet != NULL)
 143         {
 144                 stream->discardSet->free(stream->discardSet);
 145                 stream->discardSet  = NULL;
 146         }
 147         if      (stream->channelOverrides != NULL)
 148         {
 149                 stream->channelOverrides->free(stream->channelOverrides);
 150                 stream->channelOverrides = NULL;
 151         }
 152
 153         // Free our memory now
 154         //
 155         ANTLR3_FREE(stream);
 156 }
 157
 158 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
 159 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
 160 {
 161     pANTLR3_COMMON_TOKEN_STREAM stream;
 162
 163         // Create a standard token stream
 164         //
 165         stream = antlr3CommonTokenStreamSourceNew(hint, source);
 166
 167         // Install the debugger object
 168         //
 169         stream->tstream->debugger = debugger;
 170
 171         // Override standard token stream methods with debugging versions
 172         //
 173         stream->tstream->initialStreamState     = ANTLR3_FALSE;
 174
 175         stream->tstream->_LT                            = dbgTokLT;
 176
 177         stream->tstream->istream->consume               = dbgConsume;
 178         stream->tstream->istream->_LA                   = dbgLA;
 179         stream->tstream->istream->mark                  = dbgMark;
 180         stream->tstream->istream->rewind                = dbgRewindStream;
 181         stream->tstream->istream->rewindLast    = dbgRewindLast;
 182         stream->tstream->istream->seek                  = dbgSeek;
 183
 184         return stream;
 185 }
 186
 187 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
 188 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
 189 {
 190     pANTLR3_COMMON_TOKEN_STREAM stream;
 191
 192     stream = antlr3CommonTokenStreamNew(hint);
 193
 194     stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
 195
 196     stream->channelOverrides    = NULL;
 197     stream->discardSet          = NULL;
 198     stream->discardOffChannel   = ANTLR3_FALSE;
 199
 200     stream->tstream->setTokenSource(stream->tstream, source);
 201
 202     stream->free                =  antlr3CTSFree;
 203     return  stream;
 204 }
 205
 206 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
 207 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
 208 {
 209     pANTLR3_COMMON_TOKEN_STREAM stream;
 210
 211     /* Memory for the interface structure
 212      */
 213     stream  = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
 214
 215     if  (stream == NULL)
 216     {
 217         return  NULL;
 218     }
 219
 220     /* Create space for the token stream interface
 221      */
 222     stream->tstream         = antlr3TokenStreamNew();
 223     stream->tstream->super  =  stream;
 224
 225     /* Create space for the INT_STREAM interfacce
 226      */
 227     stream->tstream->istream                =  antlr3IntStreamNew();
 228     stream->tstream->istream->super         =  (stream->tstream);
 229     stream->tstream->istream->type          = ANTLR3_TOKENSTREAM;
 230
 231     /* Install the token tracking tables
 232      */
 233     stream->tokens  = antlr3VectorNew(0);
 234
 235     /* Defaults
 236      */
 237     stream->p       = -1;
 238
 239     /* Install the common token stream API
 240      */
 241     stream->setTokenTypeChannel     =  setTokenTypeChannel;
 242     stream->discardTokenType        =  discardTokenType;
 243     stream->discardOffChannelToks   =  discardOffChannel;
 244     stream->getTokens               =  getTokens;
 245     stream->getTokenRange           =  getTokenRange;
 246     stream->getTokensSet            =  getTokensSet;
 247     stream->getTokensList           =  getTokensList;
 248     stream->getTokensType           =  getTokensType;
 249
 250     /* Install the token stream API
 251      */
 252     stream->tstream->_LT                                =  tokLT;
 253     stream->tstream->get                                =  get;
 254     stream->tstream->getTokenSource             =  getTokenSource;
 255     stream->tstream->setTokenSource             =  setTokenSource;
 256     stream->tstream->toString                   =  toString;
 257     stream->tstream->toStringSS                 =  toStringSS;
 258     stream->tstream->toStringTT                 =  toStringTT;
 259         stream->tstream->setDebugListener       =  setDebugListener;
 260
 261     /* Install INT_STREAM interface
 262      */
 263     stream->tstream->istream->_LA       =  _LA;
 264     stream->tstream->istream->mark      =  mark;
 265     stream->tstream->istream->release   =  release;
 266     stream->tstream->istream->size      =  size;
 267     stream->tstream->istream->index     =  tindex;
 268     stream->tstream->istream->rewind    =  rewindStream;
 269     stream->tstream->istream->rewindLast=  rewindLast;
 270     stream->tstream->istream->seek      =  seek;
 271     stream->tstream->istream->consume   =  consume;
 272         stream->tstream->istream->getSourceName = getSourceName;
 273
 274     return  stream;
 275 }
 276
 277 // Install a debug listener adn switch to debug mode methods
 278 //
 279 static void
 280 setDebugListener        (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
 281 {
 282                 // Install the debugger object
 283         //
 284         ts->debugger = debugger;
 285
 286         // Override standard token stream methods with debugging versions
 287         //
 288         ts->initialStreamState  = ANTLR3_FALSE;
 289
 290         ts->_LT                         = dbgTokLT;
 291
 292         ts->istream->consume            = dbgConsume;
 293         ts->istream->_LA                        = dbgLA;
 294         ts->istream->mark                       = dbgMark;
 295         ts->istream->rewind                     = dbgRewindStream;
 296         ts->istream->rewindLast         = dbgRewindLast;
 297         ts->istream->seek                       = dbgSeek;
 298 }
 299
 300 /** Get the ith token from the current position 1..n where k=1 is the
 301 *  first symbol of lookahead.
 302 */
 303 static pANTLR3_COMMON_TOKEN
 304 tokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
 305 {
 306         ANTLR3_INT32    i;
 307         ANTLR3_INT32    n;
 308         pANTLR3_COMMON_TOKEN_STREAM cts;
 309
 310         cts         = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
 311
 312     if  (k < 0)
 313         {
 314                 return LB(cts, -k);
 315         }
 316
 317         if      (cts->p == -1)
 318         {
 319                 fillBuffer(cts);
 320         }
 321         if      (k == 0)
 322         {
 323                 return NULL;
 324         }
 325
 326         if      ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
 327         {
 328                 pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
 329
 330         teof->setStartIndex (teof, ts->istream->index(ts->istream));
 331         teof->setStopIndex  (teof, ts->istream->index(ts->istream));
 332                 return  teof;
 333         }
 334
 335         i       = cts->p;
 336         n       = 1;
 337
 338         /* Need to find k good tokens, skipping ones that are off channel
 339         */
 340         while   ( n < k)
 341         {
 342                 /* Skip off-channel tokens */
 343                 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token    */
 344                 n++;
 345         }
 346         if      ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
 347         {
 348                 pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
 349
 350         teof->setStartIndex (teof, ts->istream->index(ts->istream));
 351         teof->setStopIndex  (teof, ts->istream->index(ts->istream));
 352                 return  teof;
 353         }
 354
 355         // Here the token must be in the input vector. Rather then incut
 356         // function call penalty, we jsut return the pointer directly
 357         // from the vector
 358         //
 359         return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
 360         //return  (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
 361 }
 362
 363 /// Debug only method to flag consumption of initial off-channel
 364 /// tokens in the input stream
 365 ///
 366 static void
 367 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
 368 {
 369         ANTLR3_MARKER   first;
 370         ANTLR3_INT32    i;
 371         pANTLR3_TOKEN_STREAM    ts;
 372
 373         ts          = (pANTLR3_TOKEN_STREAM)        is->super;
 374         first   = is->index(is);
 375
 376         for     (i=0; i<first; i++)
 377         {
 378                 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
 379         }
 380
 381         ts->initialStreamState = ANTLR3_FALSE;
 382
 383 }
 384
 385 /// As per the normal tokLT but sends information to the debugger
 386 ///
 387 static pANTLR3_COMMON_TOKEN
 388 dbgTokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
 389 {
 390         if      (ts->initialStreamState == ANTLR3_TRUE)
 391         {
 392                 consumeInitialHiddenTokens(ts->istream);
 393         }
 394         return tokLT(ts, k);
 395 }
 396
 397 #ifdef  ANTLR3_WINDOWS
 398         /* When fully optimized VC7 complains about non reachable code.
 399          * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
 400          */
 401 #pragma warning( disable : 4702 )
 402 #endif
 403
 404 static pANTLR3_COMMON_TOKEN
 405 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
 406 {
 407     ANTLR3_INT32 i;
 408     ANTLR3_INT32 n;
 409
 410     if (cts->p == -1)
 411     {
 412         fillBuffer(cts);
 413     }
 414     if (k == 0)
 415     {
 416         return NULL;
 417     }
 418     if ((cts->p - k) < 0)
 419     {
 420         return NULL;
 421     }
 422
 423     i = cts->p;
 424     n = 1;
 425
 426     /* Need to find k good tokens, going backwards, skipping ones that are off channel
 427      */
 428     while (n <= (ANTLR3_INT32) k)
 429     {
 430         /* Skip off-channel tokens
 431          */
 432
 433         i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token    */
 434         n++;
 435     }
 436     if (i < 0)
 437     {
 438         return NULL;
 439     }
 440         // Here the token must be in the input vector. Rather then incut
 441         // function call penalty, we jsut return the pointer directly
 442         // from the vector
 443         //
 444         return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
 445 }
 446
 447 static pANTLR3_COMMON_TOKEN
 448 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
 449 {
 450     pANTLR3_COMMON_TOKEN_STREAM cts;
 451
 452     cts     = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
 453
 454     return  (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i));  /* Token index is zero based but vectors are 1 based */
 455 }
 456
 457 static pANTLR3_TOKEN_SOURCE
 458 getTokenSource  (pANTLR3_TOKEN_STREAM ts)
 459 {
 460     return  ts->tokenSource;
 461 }
 462
 463 static void
 464 setTokenSource  (   pANTLR3_TOKEN_STREAM ts,
 465                     pANTLR3_TOKEN_SOURCE tokenSource)
 466 {
 467     ts->tokenSource     = tokenSource;
 468 }
 469
 470 static pANTLR3_STRING
 471 toString    (pANTLR3_TOKEN_STREAM ts)
 472 {
 473     pANTLR3_COMMON_TOKEN_STREAM cts;
 474
 475     cts     = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
 476
 477     if  (cts->p == -1)
 478     {
 479         fillBuffer(cts);
 480     }
 481
 482     return  ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
 483 }
 484
 485 static pANTLR3_STRING
 486 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
 487 {
 488     pANTLR3_STRING string;
 489     pANTLR3_TOKEN_SOURCE tsource;
 490     pANTLR3_COMMON_TOKEN tok;
 491     ANTLR3_UINT32 i;
 492     pANTLR3_COMMON_TOKEN_STREAM cts;
 493
 494     cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 495
 496     if (cts->p == -1)
 497     {
 498         fillBuffer(cts);
 499     }
 500     if (stop >= ts->istream->size(ts->istream))
 501     {
 502         stop = ts->istream->size(ts->istream) - 1;
 503     }
 504
 505     /* Who is giving us these tokens?
 506      */
 507     tsource = ts->getTokenSource(ts);
 508
 509     if (tsource != NULL && cts->tokens != NULL)
 510     {
 511         /* Finally, let's get a string
 512          */
 513         string = tsource->strFactory->newRaw(tsource->strFactory);
 514
 515         for (i = start; i <= stop; i++)
 516         {
 517             tok = ts->get(ts, i);
 518             if (tok != NULL)
 519             {
 520                 string->appendS(string, tok->getText(tok));
 521             }
 522         }
 523
 524         return string;
 525     }
 526     return NULL;
 527
 528 }
 529
 530 static pANTLR3_STRING
 531 toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
 532 {
 533         if      (start != NULL && stop != NULL)
 534         {
 535                 return  ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
 536         }
 537         else
 538         {
 539                 return  NULL;
 540         }
 541 }
 542
 543 /** Move the input pointer to the next incoming token.  The stream
 544  *  must become active with LT(1) available.  consume() simply
 545  *  moves the input pointer so that LT(1) points at the next
 546  *  input symbol. Consume at least one token.
 547  *
 548  *  Walk past any token not on the channel the parser is listening to.
 549  */
 550 static void
 551 consume (pANTLR3_INT_STREAM is)
 552 {
 553         pANTLR3_COMMON_TOKEN_STREAM cts;
 554         pANTLR3_TOKEN_STREAM    ts;
 555
 556         ts          = (pANTLR3_TOKEN_STREAM)        is->super;
 557         cts         = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 558
 559         if      ((ANTLR3_UINT32)cts->p < cts->tokens->size(cts->tokens))
 560         {
 561                 cts->p++;
 562                 cts->p  = skipOffTokenChannels(cts, cts->p);
 563         }
 564 }
 565
 566
 567 /// As per ordinary consume but notifies the debugger about hidden
 568 /// tokens and so on.
 569 ///
 570 static void
 571 dbgConsume      (pANTLR3_INT_STREAM is)
 572 {
 573         pANTLR3_TOKEN_STREAM    ts;
 574         ANTLR3_MARKER                   a;
 575         ANTLR3_MARKER                   b;
 576         pANTLR3_COMMON_TOKEN    t;
 577
 578         ts          = (pANTLR3_TOKEN_STREAM)        is->super;
 579
 580         if      (ts->initialStreamState == ANTLR3_TRUE)
 581         {
 582                 consumeInitialHiddenTokens(is);
 583         }
 584
 585         a = is->index(is);              // Where are we right now?
 586         t = ts->_LT(ts, 1);             // Current token from stream
 587
 588         consume(is);                    // Standard consumer
 589
 590         b = is->index(is);              // Where are we after consuming 1 on channel token?
 591
 592         ts->debugger->consumeToken(ts->debugger, t);    // Tell the debugger that we consumed the first token
 593
 594         if      (b>a+1)
 595         {
 596                 // The standard consume caused the index to advance by more than 1,
 597                 // which can only happen if it skipped some off-channel tokens.
 598                 // we need to tell the debugger about those tokens.
 599                 //
 600                 ANTLR3_MARKER   i;
 601
 602                 for     (i = a+1; i<b; i++)
 603                 {
 604                         ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
 605                 }
 606
 607         }
 608 }
 609
 610 /** A simple filter mechanism whereby you can tell this token stream
 611  *  to force all tokens of type ttype to be on channel.  For example,
 612  *  when interpreting, we cannot execute actions so we need to tell
 613  *  the stream to force all WS and NEWLINE to be a different, ignored,
 614  *  channel.
 615  */
 616 static void
 617 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
 618 {
 619     if  (tokenStream->channelOverrides == NULL)
 620     {
 621         tokenStream->channelOverrides   = antlr3ListNew(10);
 622     }
 623
 624     /* We add one to the channel so we can distinguish NULL as being no entry in the
 625      * table for a particular token type.
 626      */
 627     tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
 628 }
 629
 630 static void
 631 discardTokenType    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
 632 {
 633     if  (tokenStream->discardSet == NULL)
 634     {
 635         tokenStream->discardSet = antlr3ListNew(31);
 636     }
 637
 638     /* We add one to the channel so we can distinguish NULL as being no entry in the
 639      * table for a particular token type. We could use bitsets for this I suppose too.
 640      */
 641     tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
 642 }
 643
 644 static void
 645 discardOffChannel   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
 646 {
 647     tokenStream->discardOffChannel  = discard;
 648 }
 649
 650 static pANTLR3_VECTOR
 651 getTokens   (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
 652 {
 653     if  (tokenStream->p == -1)
 654     {
 655         fillBuffer(tokenStream);
 656     }
 657
 658     return  tokenStream->tokens;
 659 }
 660
 661 static pANTLR3_LIST
 662 getTokenRange   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
 663 {
 664     return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
 665 }
 666 /** Given a start and stop index, return a List of all tokens in
 667  *  the token type BitSet.  Return null if no tokens were found.  This
 668  *  method looks at both on and off channel tokens.
 669  */
 670 static pANTLR3_LIST
 671 getTokensSet    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
 672 {
 673     pANTLR3_LIST            filteredList;
 674     ANTLR3_UINT32           i;
 675     ANTLR3_UINT32           n;
 676     pANTLR3_COMMON_TOKEN    tok;
 677
 678     if  (tokenStream->p == -1)
 679     {
 680         fillBuffer(tokenStream);
 681     }
 682     if  (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
 683     {
 684         stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
 685     }
 686     if  (start > stop)
 687     {
 688         return NULL;
 689     }
 690
 691     /* We have the range set, now we need to iterate through the
 692      * installed tokens and create a new list with just the ones we want
 693      * in it. We are just moving pointers about really.
 694      */
 695     filteredList    = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
 696
 697     for (i = start, n = 0; i<= stop; i++)
 698     {
 699         tok = tokenStream->tstream->get(tokenStream->tstream, i);
 700
 701         if  (      types == NULL
 702                 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
 703             )
 704         {
 705             filteredList->put(filteredList, n++, (void *)tok, NULL);
 706         }
 707     }
 708
 709     /* Did we get any then?
 710      */
 711     if  (filteredList->size(filteredList) == 0)
 712     {
 713         filteredList->free(filteredList);
 714         filteredList    = NULL;
 715     }
 716
 717     return  filteredList;
 718 }
 719
 720 static pANTLR3_LIST
 721 getTokensList   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
 722 {
 723     pANTLR3_BITSET  bitSet;
 724     pANTLR3_LIST    newlist;
 725
 726     bitSet  = antlr3BitsetList(list->table);
 727
 728     newlist    = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
 729
 730     bitSet->free(bitSet);
 731
 732     return  newlist;
 733
 734 }
 735
 736 static pANTLR3_LIST
 737 getTokensType   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
 738 {
 739     pANTLR3_BITSET  bitSet;
 740     pANTLR3_LIST    newlist;
 741
 742     bitSet  = antlr3BitsetOf(type, -1);
 743     newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
 744
 745     bitSet->free(bitSet);
 746
 747     return  newlist;
 748 }
 749
 750 static ANTLR3_UINT32
 751 _LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
 752 {
 753         pANTLR3_TOKEN_STREAM    ts;
 754         pANTLR3_COMMON_TOKEN    tok;
 755
 756         ts          = (pANTLR3_TOKEN_STREAM)        is->super;
 757
 758         tok         =  ts->_LT(ts, i);
 759
 760         if      (tok != NULL)
 761         {
 762                 return  tok->getType(tok);
 763         }
 764         else
 765         {
 766                 return  ANTLR3_TOKEN_INVALID;
 767         }
 768 }
 769
 770 /// As per _LA() but for debug mode.
 771 ///
 772 static ANTLR3_UINT32
 773 dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
 774 {
 775     pANTLR3_TOKEN_STREAM    ts;
 776
 777     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 778
 779         if      (ts->initialStreamState == ANTLR3_TRUE)
 780         {
 781                 consumeInitialHiddenTokens(is);
 782         }
 783         ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
 784         return  _LA(is, i);
 785 }
 786
 787 static ANTLR3_MARKER
 788 mark    (pANTLR3_INT_STREAM is)
 789 {
 790     is->lastMarker = is->index(is);
 791     return  is->lastMarker;
 792 }
 793
 794 /// As per mark() but with a call to tell the debugger we are doing this
 795 ///
 796 static ANTLR3_MARKER
 797 dbgMark (pANTLR3_INT_STREAM is)
 798 {
 799     pANTLR3_TOKEN_STREAM    ts;
 800
 801     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 802
 803         is->lastMarker = is->index(is);
 804         ts->debugger->mark(ts->debugger, is->lastMarker);
 805
 806     return  is->lastMarker;
 807 }
 808
 809 static void
 810 release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
 811 {
 812     return;
 813 }
 814
 815 static ANTLR3_UINT32
 816 size    (pANTLR3_INT_STREAM is)
 817 {
 818     pANTLR3_COMMON_TOKEN_STREAM cts;
 819     pANTLR3_TOKEN_STREAM        ts;
 820
 821     if (is->cachedSize > 0)
 822     {
 823         return  is->cachedSize;
 824     }
 825     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 826     cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 827
 828     is->cachedSize =  cts->tokens->count;
 829     return  is->cachedSize;
 830 }
 831
 832 static ANTLR3_MARKER
 833 tindex  (pANTLR3_INT_STREAM is)
 834 {
 835     pANTLR3_COMMON_TOKEN_STREAM cts;
 836     pANTLR3_TOKEN_STREAM        ts;
 837
 838     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 839     cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 840
 841     return  cts->p;
 842 }
 843
 844 static void
 845 dbgRewindLast   (pANTLR3_INT_STREAM is)
 846 {
 847         pANTLR3_TOKEN_STREAM    ts;
 848
 849     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 850
 851         ts->debugger->rewindLast(ts->debugger);
 852
 853     is->rewind(is, is->lastMarker);
 854 }
 855 static void
 856 rewindLast      (pANTLR3_INT_STREAM is)
 857 {
 858     is->rewind(is, is->lastMarker);
 859 }
 860 static void
 861 rewindStream    (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
 862 {
 863     is->seek(is, (ANTLR3_UINT32)(marker));
 864 }
 865 static void
 866 dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
 867 {
 868     pANTLR3_TOKEN_STREAM        ts;
 869
 870     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 871
 872         ts->debugger->rewind(ts->debugger, marker);
 873
 874     is->seek(is, (ANTLR3_UINT32)(marker));
 875 }
 876
 877 static void
 878 seek    (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
 879 {
 880     pANTLR3_COMMON_TOKEN_STREAM cts;
 881     pANTLR3_TOKEN_STREAM        ts;
 882
 883     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 884     cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 885
 886     cts->p  = (ANTLR3_UINT32)index;
 887 }
 888 static void
 889 dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
 890 {
 891         // TODO: Implement seek in debugger when Ter adds it to Java
 892         //
 893         seek(is, index);
 894 }
 895 ANTLR3_API void
 896 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
 897 {
 898     fillBuffer(tokenStream);
 899 }
 900 static void
 901 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
 902     ANTLR3_UINT32 index;
 903     pANTLR3_COMMON_TOKEN tok;
 904     ANTLR3_BOOLEAN discard;
 905     void * channelI;
 906
 907     /* Start at index 0 of course
 908      */
 909     index = 0;
 910
 911     /* Pick out the next token from the token source
 912      * Remember we just get a pointer (reference if you like) here
 913      * and so if we store it anywhere, we don't set any pointers to auto free it.
 914      */
 915     tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
 916
 917     while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
 918     {
 919         discard = ANTLR3_FALSE; /* Assume we are not discarding */
 920
 921         /* I employ a bit of a trick, or perhaps hack here. Rather than
 922          * store a pointer to a structure in the override map and discard set
 923          * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
 924          * we can distinguish "not being there" from "being channel or type 0"
 925          */
 926
 927         if (tokenStream->discardSet != NULL
 928             && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
 929         {
 930             discard = ANTLR3_TRUE;
 931         }
 932         else if (   tokenStream->discardOffChannel == ANTLR3_TRUE
 933                  && tok->getChannel(tok) != tokenStream->channel
 934                  )
 935         {
 936             discard = ANTLR3_TRUE;
 937         }
 938         else if (tokenStream->channelOverrides != NULL)
 939         {
 940             /* See if this type is in the override map
 941              */
 942             channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
 943
 944             if (channelI != NULL)
 945             {
 946                 /* Override found
 947                  */
 948                 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
 949             }
 950         }
 951
 952         /* If not discarding it, add it to the list at the current index
 953          */
 954         if (discard == ANTLR3_FALSE)
 955         {
 956             /* Add it, indicating that we will delete it and the table should not
 957              */
 958             tok->setTokenIndex(tok, index);
 959             tokenStream->p++;
 960             tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
 961             index++;
 962         }
 963
 964         tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
 965     }
 966
 967     /* Cache the size so we don't keep doing indirect method calls. We do this as
 968      * early as possible so that anything after this may utilize the cached value.
 969      */
 970     tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
 971
 972     /* Set the consume pointer to the first token that is on our channel
 973      */
 974     tokenStream->p = 0;
 975     tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
 976
 977 }
 978
 979 /// Given a starting index, return the index of the first on-channel
 980 ///  token.
 981 ///
 982 static ANTLR3_UINT32
 983 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
 984     ANTLR3_INT32 n;
 985     pANTLR3_COMMON_TOKEN tok;
 986
 987     n = tokenStream->tstream->istream->cachedSize;
 988
 989     while (i < n)
 990     {
 991         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
 992
 993         if (tok->channel!= tokenStream->channel)
 994         {
 995             i++;
 996         }
 997         else
 998         {
 999             return i;
1000         }
1001     }
1002     return i;
1003 }
1004
1005 static ANTLR3_UINT32
1006 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
1007 {
1008     pANTLR3_COMMON_TOKEN tok;
1009
1010     while (x >= 0)
1011     {
1012         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
1013
1014         if ((tok->channel != tokenStream->channel))
1015         {
1016             x--;
1017         }
1018         else
1019         {
1020             return x;
1021         }
1022     }
1023     return x;
1024 }
1025
1026 /// Return a string that represents the name assoicated with the input source
1027 ///
1028 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
1029 ///
1030 /// /returns
1031 /// /implements ANTLR3_INT_STREAM_struct::getSourceName()
1032 ///
1033 static pANTLR3_STRING
1034 getSourceName                           (pANTLR3_INT_STREAM is)
1035 {
1036         // Slightly convoluted as we must trace back to the lexer's input source
1037         // via the token source. The streamName that is here is not initialized
1038         // because this is a token stream, not a file or string stream, which are the
1039         // only things that have a context for a source name.
1040         //
1041         return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
1042 }