thirdparty/libantlr3c-3.2/src/antlr3tokenstream.c

   1 /// \file
   2 /// Default implementation of CommonTokenStream
   3 ///
   4
   5 // [The "BSD licence"]
   6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
   7 // http://www.temporal-wave.com
   8 // http://www.linkedin.com/in/jimidle
   9 //
  10 // All rights reserved.
  11 //
  12 // Redistribution and use in source and binary forms, with or without
  13 // modification, are permitted provided that the following conditions
  14 // are met:
  15 // 1. Redistributions of source code must retain the above copyright
  16 //    notice, this list of conditions and the following disclaimer.
  17 // 2. Redistributions in binary form must reproduce the above copyright
  18 //    notice, this list of conditions and the following disclaimer in the
  19 //    documentation and/or other materials provided with the distribution.
  20 // 3. The name of the author may not be used to endorse or promote products
  21 //    derived from this software without specific prior written permission.
  22 //
  23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33
  34 #include    <antlr3tokenstream.h>
  35
  36 #ifdef  ANTLR3_WINDOWS
  37 #pragma warning( disable : 4100 )
  38 #endif
  39
  40 // COMMON_TOKEN_STREAM API
  41 //
  42 static void                                     setTokenTypeChannel     (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
  43 static void                                     discardTokenType        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
  44 static void                                     discardOffChannel       (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
  45 static pANTLR3_VECTOR           getTokens                       (pANTLR3_COMMON_TOKEN_STREAM cts);
  46 static pANTLR3_LIST                     getTokenRange           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
  47 static pANTLR3_LIST                     getTokensSet            (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
  48 static pANTLR3_LIST                     getTokensList           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
  49 static pANTLR3_LIST                     getTokensType           (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
  50
  51 // TOKEN_STREAM API
  52 //
  53 static pANTLR3_COMMON_TOKEN get                                 (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
  54 static pANTLR3_TOKEN_SOURCE getTokenSource              (pANTLR3_TOKEN_STREAM ts);
  55 static void                                     setTokenSource          (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
  56 static pANTLR3_STRING       toString                    (pANTLR3_TOKEN_STREAM ts);
  57 static pANTLR3_STRING       toStringSS                  (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
  58 static pANTLR3_STRING       toStringTT                  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
  59 static void                                     setDebugListener        (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
  60
  61 // INT STREAM API
  62 //
  63 static void                                     consume                                         (pANTLR3_INT_STREAM is);
  64 static void                                     dbgConsume                                      (pANTLR3_INT_STREAM is);
  65 static ANTLR3_UINT32        _LA                                                 (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
  66 static ANTLR3_UINT32        dbgLA                                               (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
  67 static ANTLR3_MARKER        mark                                                (pANTLR3_INT_STREAM is);
  68 static ANTLR3_MARKER        dbgMark                                             (pANTLR3_INT_STREAM is);
  69 static void                                     release                                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
  70 static ANTLR3_UINT32        size                                                (pANTLR3_INT_STREAM is);
  71 static ANTLR3_MARKER            tindex                                          (pANTLR3_INT_STREAM is);
  72 static void                                     rewindStream                            (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
  73 static void                                     dbgRewindStream                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
  74 static void                                     rewindLast                                      (pANTLR3_INT_STREAM is);
  75 static void                                     dbgRewindLast                           (pANTLR3_INT_STREAM is);
  76 static void                                     seek                                            (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
  77 static void                                     dbgSeek                                         (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
  78 static pANTLR3_STRING           getSourceName                           (pANTLR3_INT_STREAM is);
  79 static void                                     antlr3TokenStreamFree           (pANTLR3_TOKEN_STREAM       stream);
  80 static void                                     antlr3CTSFree                           (pANTLR3_COMMON_TOKEN_STREAM    stream);
  81
  82 // Helpers
  83 //
  84 static void                                     fillBuffer                                      (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
  85 static ANTLR3_UINT32        skipOffTokenChannels                (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
  86 static ANTLR3_UINT32        skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
  87 static pANTLR3_COMMON_TOKEN LB                                                  (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
  88
  89 ANTLR3_API pANTLR3_TOKEN_STREAM
  90 antlr3TokenStreamNew()
  91 {
  92     pANTLR3_TOKEN_STREAM stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
  93
  94     if (stream == NULL)
  95         return NULL;
  96
  97     // Install basic API
  98     //
  99     stream->free = antlr3TokenStreamFree;
 100     return stream;
 101 }
 102
 103 static void
 104 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
 105 {
 106     ANTLR3_FREE(stream);
 107 }
 108
 109 static void
 110 antlr3CTSFree(pANTLR3_COMMON_TOKEN_STREAM stream)
 111 {
 112         // We only free up our subordinate interfaces if they belong
 113         // to us, otherwise we let whoever owns them deal with them.
 114         //
 115         if      (stream->tstream->super == stream)
 116         {
 117                 if      (stream->tstream->istream->super == stream->tstream)
 118                 {
 119                         stream->tstream->istream->free(stream->tstream->istream);
 120                         stream->tstream->istream = NULL;
 121                 }
 122                 stream->tstream->free(stream->tstream);
 123         }
 124
 125         // Now we free our own resources
 126         //
 127         if      (stream->tokens != NULL)
 128         {
 129                 vectorFree(stream->tokens);
 130                 stream->tokens  = NULL;
 131         }
 132         if      (stream->discardSet != NULL)
 133         {
 134                 listFree(stream->discardSet);
 135                 stream->discardSet  = NULL;
 136         }
 137         if      (stream->channelOverrides != NULL)
 138         {
 139                 listFree(stream->channelOverrides);
 140                 stream->channelOverrides = NULL;
 141         }
 142
 143         // Free our memory now
 144         //
 145         ANTLR3_FREE(stream);
 146 }
 147
 148 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
 149 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
 150 {
 151     pANTLR3_COMMON_TOKEN_STREAM stream;
 152
 153         // Create a standard token stream
 154         //
 155         stream = antlr3CommonTokenStreamSourceNew(hint, source);
 156
 157         // Install the debugger object
 158         //
 159         stream->tstream->debugger = debugger;
 160
 161         // Override standard token stream methods with debugging versions
 162         //
 163         stream->tstream->initialStreamState     = ANTLR3_FALSE;
 164
 165         stream->tstream->istream->consume               = dbgConsume;
 166         stream->tstream->istream->_LA                   = dbgLA;
 167         stream->tstream->istream->mark                  = dbgMark;
 168         stream->tstream->istream->rewind                = dbgRewindStream;
 169         stream->tstream->istream->rewindLast    = dbgRewindLast;
 170         stream->tstream->istream->seek                  = dbgSeek;
 171
 172         return stream;
 173 }
 174
 175 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
 176 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
 177 {
 178     pANTLR3_COMMON_TOKEN_STREAM stream;
 179
 180     stream = antlr3CommonTokenStreamNew(hint);
 181
 182     stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
 183
 184     stream->channelOverrides    = NULL;
 185     stream->discardSet          = NULL;
 186     stream->discardOffChannel   = ANTLR3_FALSE;
 187
 188     stream->tstream->setTokenSource(stream->tstream, source);
 189
 190     stream->free                =  antlr3CTSFree;
 191     return  stream;
 192 }
 193
 194 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
 195 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
 196 {
 197     pANTLR3_COMMON_TOKEN_STREAM stream;
 198
 199     /* Memory for the interface structure
 200      */
 201     stream  = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
 202
 203     if  (stream == NULL)
 204     {
 205         return  NULL;
 206     }
 207
 208     /* Create space for the token stream interface
 209      */
 210     stream->tstream         = antlr3TokenStreamNew();
 211     stream->tstream->super  =  stream;
 212
 213     /* Create space for the INT_STREAM interfacce
 214      */
 215     stream->tstream->istream                =  antlr3IntStreamNew();
 216     stream->tstream->istream->super         =  (stream->tstream);
 217     stream->tstream->istream->type          = ANTLR3_TOKENSTREAM;
 218
 219     /* Install the token tracking tables
 220      */
 221     stream->tokens  = antlr3VectorNew(0);
 222
 223     /* Defaults
 224      */
 225     stream->p       = -1;
 226
 227     /* Install the common token stream API
 228      */
 229     stream->setTokenTypeChannel     =  setTokenTypeChannel;
 230     stream->discardTokenType        =  discardTokenType;
 231     stream->discardOffChannelToks   =  discardOffChannel;
 232     stream->getTokens               =  getTokens;
 233     stream->getTokenRange           =  getTokenRange;
 234     stream->getTokensSet            =  getTokensSet;
 235     stream->getTokensList           =  getTokensList;
 236     stream->getTokensType           =  getTokensType;
 237
 238     /* Install the token stream API
 239      */
 240     stream->tstream->get                                =  get;
 241     stream->tstream->getTokenSource             =  getTokenSource;
 242     stream->tstream->setTokenSource             =  setTokenSource;
 243     stream->tstream->toString                   =  toString;
 244     stream->tstream->toStringSS                 =  toStringSS;
 245     stream->tstream->toStringTT                 =  toStringTT;
 246         stream->tstream->setDebugListener       =  setDebugListener;
 247
 248     /* Install INT_STREAM interface
 249      */
 250     stream->tstream->istream->_LA       =  _LA;
 251     stream->tstream->istream->mark      =  mark;
 252     stream->tstream->istream->release   =  release;
 253     stream->tstream->istream->size      =  size;
 254     stream->tstream->istream->index     =  tindex;
 255     stream->tstream->istream->rewind    =  rewindStream;
 256     stream->tstream->istream->rewindLast=  rewindLast;
 257     stream->tstream->istream->seek      =  seek;
 258     stream->tstream->istream->consume   =  consume;
 259         stream->tstream->istream->getSourceName = getSourceName;
 260
 261     return  stream;
 262 }
 263
 264 // Install a debug listener adn switch to debug mode methods
 265 //
 266 static void
 267 setDebugListener        (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
 268 {
 269                 // Install the debugger object
 270         //
 271         ts->debugger = debugger;
 272
 273         // Override standard token stream methods with debugging versions
 274         //
 275         ts->initialStreamState  = ANTLR3_FALSE;
 276
 277         ts->istream->consume            = dbgConsume;
 278         ts->istream->_LA                        = dbgLA;
 279         ts->istream->mark                       = dbgMark;
 280         ts->istream->rewind                     = dbgRewindStream;
 281         ts->istream->rewindLast         = dbgRewindLast;
 282         ts->istream->seek                       = dbgSeek;
 283 }
 284
 285 /** Get the ith token from the current position 1..n where k=1 is the
 286 *  first symbol of lookahead.
 287 */
 288 pANTLR3_COMMON_TOKEN
 289 tokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
 290 {
 291     pANTLR3_COMMON_TOKEN_STREAM cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
 292
 293     if (k < 0)
 294         return LB(cts, -k);
 295
 296     if (cts->p == -1)
 297         fillBuffer(cts);
 298
 299     if (k == 0)
 300         return NULL;
 301
 302     if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
 303     {
 304         pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
 305
 306         teof->start = ts->istream->index(ts->istream);
 307         teof->stop  = ts->istream->index(ts->istream);
 308         return  teof;
 309     }
 310
 311     {
 312         ANTLR3_INT32    i = cts->p;
 313         ANTLR3_INT32    n = 1;
 314
 315         /* Need to find k good tokens, skipping ones that are off channel */
 316         while ( n < k )
 317         {
 318             /* Skip off-channel tokens */
 319             i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
 320             n++;
 321         }
 322         if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
 323         {
 324             pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
 325
 326             teof->start = ts->istream->index(ts->istream);
 327             teof->stop  = ts->istream->index(ts->istream);
 328             return  teof;
 329         }
 330
 331         // Here the token must be in the input vector. Rather then incut
 332         // function call penalty, we just return the pointer directly
 333         // from the vector
 334         //
 335         return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
 336     }
 337 }
 338
 339 /// Debug only method to flag consumption of initial off-channel
 340 /// tokens in the input stream
 341 ///
 342 static void
 343 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
 344 {
 345         ANTLR3_MARKER   first;
 346         ANTLR3_INT32    i;
 347         pANTLR3_TOKEN_STREAM    ts;
 348
 349         ts          = (pANTLR3_TOKEN_STREAM)        is->super;
 350         first   = is->index(is);
 351
 352         for     (i=0; i<first; i++)
 353         {
 354                 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
 355         }
 356
 357         ts->initialStreamState = ANTLR3_FALSE;
 358
 359 }
 360
 361 #ifdef  ANTLR3_WINDOWS
 362         /* When fully optimized VC7 complains about non reachable code.
 363          * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
 364          */
 365 #pragma warning( disable : 4702 )
 366 #endif
 367
 368 static pANTLR3_COMMON_TOKEN
 369 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
 370 {
 371     ANTLR3_INT32 i;
 372     ANTLR3_INT32 n;
 373
 374     if (cts->p == -1)
 375     {
 376         fillBuffer(cts);
 377     }
 378     if (k == 0)
 379     {
 380         return NULL;
 381     }
 382     if ((cts->p - k) < 0)
 383     {
 384         return NULL;
 385     }
 386
 387     i = cts->p;
 388     n = 1;
 389
 390     /* Need to find k good tokens, going backwards, skipping ones that are off channel
 391      */
 392     while (n <= (ANTLR3_INT32) k)
 393     {
 394         /* Skip off-channel tokens
 395          */
 396
 397         i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token    */
 398         n++;
 399     }
 400     if (i < 0)
 401     {
 402         return NULL;
 403     }
 404         // Here the token must be in the input vector. Rather then incut
 405         // function call penalty, we jsut return the pointer directly
 406         // from the vector
 407         //
 408         return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
 409 }
 410
 411 static pANTLR3_COMMON_TOKEN
 412 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
 413 {
 414     pANTLR3_COMMON_TOKEN_STREAM cts;
 415
 416     cts     = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
 417
 418     return  (pANTLR3_COMMON_TOKEN)(vectorGet(cts->tokens, i));  /* Token index is zero based but vectors are 1 based */
 419 }
 420
 421 static pANTLR3_TOKEN_SOURCE
 422 getTokenSource  (pANTLR3_TOKEN_STREAM ts)
 423 {
 424     return  ts->tokenSource;
 425 }
 426
 427 static void
 428 setTokenSource  (   pANTLR3_TOKEN_STREAM ts,
 429                     pANTLR3_TOKEN_SOURCE tokenSource)
 430 {
 431     ts->tokenSource     = tokenSource;
 432 }
 433
 434 static pANTLR3_STRING
 435 toString    (pANTLR3_TOKEN_STREAM ts)
 436 {
 437     pANTLR3_COMMON_TOKEN_STREAM cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
 438
 439     if(cts->p == -1)
 440         fillBuffer(cts);
 441
 442     return  ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
 443 }
 444
 445 static pANTLR3_STRING
 446 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
 447 {
 448     pANTLR3_STRING string;
 449     pANTLR3_TOKEN_SOURCE tsource;
 450     pANTLR3_COMMON_TOKEN tok;
 451     ANTLR3_UINT32 i;
 452     pANTLR3_COMMON_TOKEN_STREAM cts;
 453
 454     cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 455
 456     if (cts->p == -1)
 457     {
 458         fillBuffer(cts);
 459     }
 460     if (stop >= ts->istream->size(ts->istream))
 461     {
 462         stop = ts->istream->size(ts->istream) - 1;
 463     }
 464
 465     /* Who is giving us these tokens?
 466      */
 467     tsource = ts->getTokenSource(ts);
 468
 469     if (tsource != NULL && cts->tokens != NULL)
 470     {
 471         /* Finally, let's get a string
 472          */
 473         string = tsource->strFactory->newRaw(tsource->strFactory);
 474
 475         for (i = start; i <= stop; i++)
 476         {
 477             tok = ts->get(ts, i);
 478             if (tok != NULL)
 479             {
 480                 string->appendS(string, tok->getText(tok));
 481             }
 482         }
 483
 484         return string;
 485     }
 486     return NULL;
 487
 488 }
 489
 490 static pANTLR3_STRING
 491 toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
 492 {
 493     if (start != NULL && stop != NULL)
 494         return ts->toStringSS(ts, (ANTLR3_UINT32)start->index, (ANTLR3_UINT32)stop->index);
 495     return NULL;
 496 }
 497
 498 /** Move the input pointer to the next incoming token.  The stream
 499  *  must become active with LT(1) available.  consume() simply
 500  *  moves the input pointer so that LT(1) points at the next
 501  *  input symbol. Consume at least one token.
 502  *
 503  *  Walk past any token not on the channel the parser is listening to.
 504  */
 505 static void
 506 consume(pANTLR3_INT_STREAM is)
 507 {
 508     pANTLR3_TOKEN_STREAM        ts = (pANTLR3_TOKEN_STREAM) is->super;
 509     pANTLR3_COMMON_TOKEN_STREAM cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 510
 511     if ((ANTLR3_UINT32)cts->p < cts->tokens->count)
 512     {
 513         cts->p++;
 514         cts->p = skipOffTokenChannels(cts, cts->p);
 515     }
 516 }
 517
 518
 519 /// As per ordinary consume but notifies the debugger about hidden
 520 /// tokens and so on.
 521 ///
 522 static void
 523 dbgConsume      (pANTLR3_INT_STREAM is)
 524 {
 525         pANTLR3_TOKEN_STREAM    ts = (pANTLR3_TOKEN_STREAM)is->super;
 526         ANTLR3_MARKER                   a;
 527         ANTLR3_MARKER                   b;
 528         pANTLR3_COMMON_TOKEN    t;
 529
 530         if      (ts->initialStreamState == ANTLR3_TRUE)
 531         {
 532                 consumeInitialHiddenTokens(is);
 533         }
 534
 535         a = is->index(is);              // Where are we right now?
 536         t = tokLT(ts, 1);               // Current token from stream
 537
 538         consume(is);                    // Standard consumer
 539
 540         b = is->index(is);              // Where are we after consuming 1 on channel token?
 541
 542         ts->debugger->consumeToken(ts->debugger, t);    // Tell the debugger that we consumed the first token
 543
 544         if      (b>a+1)
 545         {
 546                 // The standard consume caused the index to advance by more than 1,
 547                 // which can only happen if it skipped some off-channel tokens.
 548                 // we need to tell the debugger about those tokens.
 549                 //
 550                 ANTLR3_MARKER   i;
 551
 552                 for     (i = a+1; i<b; i++)
 553                 {
 554                         ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
 555                 }
 556
 557         }
 558 }
 559
 560 /** A simple filter mechanism whereby you can tell this token stream
 561  *  to force all tokens of type ttype to be on channel.  For example,
 562  *  when interpreting, we cannot execute actions so we need to tell
 563  *  the stream to force all WS and NEWLINE to be a different, ignored,
 564  *  channel.
 565  */
 566 static void
 567 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
 568 {
 569     if  (tokenStream->channelOverrides == NULL)
 570     {
 571         tokenStream->channelOverrides   = antlr3ListNew(10);
 572     }
 573
 574     /* We add one to the channel so we can distinguish NULL as being no entry in the
 575      * table for a particular token type.
 576      */
 577     listPut(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
 578 }
 579
 580 static void
 581 discardTokenType    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
 582 {
 583     if  (tokenStream->discardSet == NULL)
 584     {
 585         tokenStream->discardSet = antlr3ListNew(31);
 586     }
 587
 588     /* We add one to the channel so we can distinguish NULL as being no entry in the
 589      * table for a particular token type. We could use bitsets for this I suppose too.
 590      */
 591     listPut(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
 592 }
 593
 594 static void
 595 discardOffChannel   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
 596 {
 597     tokenStream->discardOffChannel  = discard;
 598 }
 599
 600 static pANTLR3_VECTOR
 601 getTokens   (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
 602 {
 603     if (tokenStream->p == -1)
 604         fillBuffer(tokenStream);
 605
 606     return  tokenStream->tokens;
 607 }
 608
 609 static pANTLR3_LIST
 610 getTokenRange   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
 611 {
 612     return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
 613 }
 614 /** Given a start and stop index, return a List of all tokens in
 615  *  the token type BitSet.  Return null if no tokens were found.  This
 616  *  method looks at both on and off channel tokens.
 617  */
 618 static pANTLR3_LIST
 619 getTokensSet    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
 620 {
 621     pANTLR3_LIST            filteredList;
 622     ANTLR3_UINT32           i;
 623     ANTLR3_UINT32           n;
 624     pANTLR3_COMMON_TOKEN    tok;
 625
 626     if  (tokenStream->p == -1)
 627     {
 628         fillBuffer(tokenStream);
 629     }
 630     if  (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
 631     {
 632         stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
 633     }
 634     if  (start > stop)
 635     {
 636         return NULL;
 637     }
 638
 639     /* We have the range set, now we need to iterate through the
 640      * installed tokens and create a new list with just the ones we want
 641      * in it. We are just moving pointers about really.
 642      */
 643     filteredList    = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
 644
 645     for (i = start, n = 0; i<= stop; i++)
 646     {
 647         tok = tokenStream->tstream->get(tokenStream->tstream, i);
 648
 649         if  (      types == NULL
 650                 || types->isMember(types, tok->type == ANTLR3_TRUE)
 651             )
 652         {
 653             listPut(filteredList, n++, (void *)tok, NULL);
 654         }
 655     }
 656
 657     /* Did we get any then?
 658      */
 659     if  (listSize(filteredList) == 0)
 660     {
 661         listFree(filteredList);
 662         filteredList    = NULL;
 663     }
 664
 665     return  filteredList;
 666 }
 667
 668 static pANTLR3_LIST
 669 getTokensList   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
 670 {
 671     pANTLR3_BITSET  bitSet;
 672     pANTLR3_LIST    newlist;
 673
 674     bitSet  = antlr3BitsetList(list->table);
 675
 676     newlist    = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
 677
 678     bitSet->free(bitSet);
 679
 680     return  newlist;
 681
 682 }
 683
 684 static pANTLR3_LIST
 685 getTokensType(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
 686 {
 687     pANTLR3_BITSET  bitSet = antlr3BitsetOf(type, -1);
 688     pANTLR3_LIST    newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
 689
 690     bitSet->free(bitSet);
 691     return  newlist;
 692 }
 693
 694 static ANTLR3_UINT32
 695 _LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
 696 {
 697     pANTLR3_TOKEN_STREAM    ts = (pANTLR3_TOKEN_STREAM) is->super;
 698     pANTLR3_COMMON_TOKEN    tok = tokLT(ts, i);
 699
 700     if(tok != NULL)
 701         return tok->type;
 702     return ANTLR3_TOKEN_INVALID;
 703 }
 704
 705 /// As per _LA() but for debug mode.
 706 ///
 707 static ANTLR3_UINT32
 708 dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
 709 {
 710     pANTLR3_TOKEN_STREAM    ts;
 711
 712     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 713
 714         if      (ts->initialStreamState == ANTLR3_TRUE)
 715         {
 716                 consumeInitialHiddenTokens(is);
 717         }
 718         ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
 719         return  _LA(is, i);
 720 }
 721
 722 static ANTLR3_MARKER
 723 mark(pANTLR3_INT_STREAM is)
 724 {
 725     is->lastMarker = is->index(is);
 726     return is->lastMarker;
 727 }
 728
 729 /// As per mark() but with a call to tell the debugger we are doing this
 730 ///
 731 static ANTLR3_MARKER
 732 dbgMark (pANTLR3_INT_STREAM is)
 733 {
 734     pANTLR3_TOKEN_STREAM    ts;
 735
 736     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 737
 738         is->lastMarker = is->index(is);
 739         ts->debugger->mark(ts->debugger, is->lastMarker);
 740
 741     return  is->lastMarker;
 742 }
 743
 744 static void
 745 release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
 746 {
 747     return;
 748 }
 749
 750 static ANTLR3_UINT32
 751 size(pANTLR3_INT_STREAM is)
 752 {
 753     if (is->cachedSize > 0)
 754         return  is->cachedSize;
 755
 756     {
 757         pANTLR3_COMMON_TOKEN_STREAM cts;
 758         pANTLR3_TOKEN_STREAM ts;
 759
 760         ts = (pANTLR3_TOKEN_STREAM) is->super;
 761         cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 762
 763         is->cachedSize = cts->tokens->count;
 764     }
 765     return  is->cachedSize;
 766 }
 767
 768 static ANTLR3_MARKER
 769 tindex(pANTLR3_INT_STREAM is)
 770 {
 771     pANTLR3_TOKEN_STREAM        ts = (pANTLR3_TOKEN_STREAM) is->super;
 772     pANTLR3_COMMON_TOKEN_STREAM cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 773
 774     return  cts->p;
 775 }
 776
 777 static void
 778 dbgRewindLast   (pANTLR3_INT_STREAM is)
 779 {
 780         pANTLR3_TOKEN_STREAM    ts;
 781
 782     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 783
 784         ts->debugger->rewindLast(ts->debugger);
 785
 786     is->rewind(is, is->lastMarker);
 787 }
 788 static void
 789 rewindLast      (pANTLR3_INT_STREAM is)
 790 {
 791     is->rewind(is, is->lastMarker);
 792 }
 793 static void
 794 rewindStream    (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
 795 {
 796     is->seek(is, (ANTLR3_UINT32)(marker));
 797 }
 798 static void
 799 dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
 800 {
 801     pANTLR3_TOKEN_STREAM        ts;
 802
 803     ts      = (pANTLR3_TOKEN_STREAM)        is->super;
 804
 805         ts->debugger->rewind(ts->debugger, marker);
 806
 807     is->seek(is, (ANTLR3_UINT32)(marker));
 808 }
 809
 810 static void
 811 seek(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
 812 {
 813     pANTLR3_TOKEN_STREAM        ts = (pANTLR3_TOKEN_STREAM) is->super;
 814     pANTLR3_COMMON_TOKEN_STREAM cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
 815
 816     cts->p  = (ANTLR3_UINT32)index;
 817 }
 818 static void
 819 dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
 820 {
 821         // TODO: Implement seek in debugger when Ter adds it to Java
 822         //
 823         seek(is, index);
 824 }
 825 ANTLR3_API void
 826 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
 827 {
 828     fillBuffer(tokenStream);
 829 }
 830 static void
 831 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
 832     ANTLR3_UINT32 index = 0; /* Start at index 0 of course */
 833     pANTLR3_COMMON_TOKEN tok;
 834     ANTLR3_BOOLEAN discard;
 835     void * channelI;
 836
 837
 838     /* Pick out the next token from the token source
 839      * Remember we just get a pointer (reference if you like) here
 840      * and so if we store it anywhere, we don't set any pointers to auto free it.
 841      */
 842     tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
 843
 844     while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
 845     {
 846         discard = ANTLR3_FALSE; /* Assume we are not discarding */
 847
 848         /* I employ a bit of a trick, or perhaps hack here. Rather than
 849          * store a pointer to a structure in the override map and discard set
 850          * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
 851          * we can distinguish "not being there" from "being channel or type 0"
 852          */
 853
 854         if (tokenStream->discardSet != NULL
 855             && listGet(tokenStream->discardSet, tok->type) != NULL)
 856         {
 857             discard = ANTLR3_TRUE;
 858         }
 859         else if (   tokenStream->discardOffChannel == ANTLR3_TRUE
 860                  && tok->channel != tokenStream->channel
 861                  )
 862         {
 863             discard = ANTLR3_TRUE;
 864         }
 865         else if (tokenStream->channelOverrides != NULL)
 866         {
 867             /* See if this type is in the override map */
 868             channelI = listGet(tokenStream->channelOverrides, tok->type + 1);
 869
 870             if (channelI != NULL)
 871             {
 872                 /* Override found */
 873                 tok->channel = ANTLR3_UINT32_CAST(channelI) - 1;
 874             }
 875         }
 876
 877         /* If not discarding it, add it to the list at the current index */
 878         if (discard == ANTLR3_FALSE)
 879         {
 880             /* Add it, indicating that we will delete it and the table should not */
 881             tok->index = index;
 882             tokenStream->p++;
 883             vectorAdd(tokenStream->tokens, (void *) tok, NULL);
 884             index++;
 885         }
 886
 887         tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
 888     }
 889
 890     /* Cache the size so we don't keep doing indirect method calls. We do this as
 891      * early as possible so that anything after this may utilize the cached value.
 892      */
 893     tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
 894
 895     /* Set the consume pointer to the first token that is on our channel
 896      */
 897     tokenStream->p = 0;
 898     tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
 899
 900 }
 901
 902 /// Given a starting index, return the index of the first on-channel
 903 ///  token.
 904 ///
 905 static ANTLR3_UINT32
 906 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
 907     ANTLR3_INT32 n;
 908     pANTLR3_COMMON_TOKEN tok;
 909
 910     n = tokenStream->tstream->istream->cachedSize;
 911
 912     while (i < n)
 913     {
 914         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
 915
 916         if (tok->channel!= tokenStream->channel)
 917         {
 918             i++;
 919         }
 920         else
 921         {
 922             return i;
 923         }
 924     }
 925     return i;
 926 }
 927
 928 static ANTLR3_UINT32
 929 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
 930 {
 931     pANTLR3_COMMON_TOKEN tok;
 932
 933     while (x >= 0)
 934     {
 935         tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
 936
 937         if ((tok->channel != tokenStream->channel))
 938         {
 939             x--;
 940         }
 941         else
 942         {
 943             return x;
 944         }
 945     }
 946     return x;
 947 }
 948
 949 /// Return a string that represents the name assoicated with the input source
 950 ///
 951 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
 952 ///
 953 /// /returns
 954 /// /implements ANTLR3_INT_STREAM_struct::getSourceName()
 955 ///
 956 static pANTLR3_STRING
 957 getSourceName                           (pANTLR3_INT_STREAM is)
 958 {
 959         // Slightly convoluted as we must trace back to the lexer's input source
 960         // via the token source. The streamName that is here is not initialized
 961         // because this is a token stream, not a file or string stream, which are the
 962         // only things that have a context for a source name.
 963         //
 964         return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
 965 }