clang/lib/Lex/PPCaching.cpp

   1 //===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements pieces of the Preprocessor interface that manage the
  10 // caching of lexed tokens.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "clang/Lex/Preprocessor.h"
  15 using namespace clang;
  16
  17 // EnableBacktrackAtThisPos - From the point that this method is called, and
  18 // until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
  19 // keeps track of the lexed tokens so that a subsequent Backtrack() call will
  20 // make the Preprocessor re-lex the same tokens.
  21 //
  22 // Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
  23 // be called multiple times and CommitBacktrackedTokens/Backtrack calls will
  24 // be combined with the EnableBacktrackAtThisPos calls in reverse order.
  25 void Preprocessor::EnableBacktrackAtThisPos() {
  26   assert(LexLevel == 0 && "cannot use lookahead while lexing");
  27   BacktrackPositions.push_back(CachedLexPos);
  28   EnterCachingLexMode();
  29 }
  30
  31 // Disable the last EnableBacktrackAtThisPos call.
  32 void Preprocessor::CommitBacktrackedTokens() {
  33   assert(!BacktrackPositions.empty()
  34          && "EnableBacktrackAtThisPos was not called!");
  35   BacktrackPositions.pop_back();
  36 }
  37
  38 // Make Preprocessor re-lex the tokens that were lexed since
  39 // EnableBacktrackAtThisPos() was previously called.
  40 void Preprocessor::Backtrack() {
  41   assert(!BacktrackPositions.empty()
  42          && "EnableBacktrackAtThisPos was not called!");
  43   CachedLexPos = BacktrackPositions.back();
  44   BacktrackPositions.pop_back();
  45   recomputeCurLexerKind();
  46 }
  47
  48 void Preprocessor::CachingLex(Token &Result) {
  49   if (!InCachingLexMode())
  50     return;
  51
  52   // The assert in EnterCachingLexMode should prevent this from happening.
  53   assert(LexLevel == 1 &&
  54          "should not use token caching within the preprocessor");
  55
  56   if (CachedLexPos < CachedTokens.size()) {
  57     Result = CachedTokens[CachedLexPos++];
  58     Result.setFlag(Token::IsReinjected);
  59     return;
  60   }
  61
  62   ExitCachingLexMode();
  63   Lex(Result);
  64
  65   if (isBacktrackEnabled()) {
  66     // Cache the lexed token.
  67     EnterCachingLexModeUnchecked();
  68     CachedTokens.push_back(Result);
  69     ++CachedLexPos;
  70     return;
  71   }
  72
  73   if (CachedLexPos < CachedTokens.size()) {
  74     EnterCachingLexModeUnchecked();
  75   } else {
  76     // All cached tokens were consumed.
  77     CachedTokens.clear();
  78     CachedLexPos = 0;
  79   }
  80 }
  81
  82 void Preprocessor::EnterCachingLexMode() {
  83   // The caching layer sits on top of all the other lexers, so it's incorrect
  84   // to cache tokens while inside a nested lex action. The cached tokens would
  85   // be retained after returning to the enclosing lex action and, at best,
  86   // would appear at the wrong position in the token stream.
  87   assert(LexLevel == 0 &&
  88          "entered caching lex mode while lexing something else");
  89
  90   if (InCachingLexMode()) {
  91     assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
  92     return;
  93   }
  94
  95   EnterCachingLexModeUnchecked();
  96 }
  97
  98 void Preprocessor::EnterCachingLexModeUnchecked() {
  99   assert(CurLexerKind != CLK_CachingLexer && "already in caching lex mode");
 100   PushIncludeMacroStack();
 101   CurLexerKind = CLK_CachingLexer;
 102 }
 103
 104
 105 const Token &Preprocessor::PeekAhead(unsigned N) {
 106   assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
 107   ExitCachingLexMode();
 108   for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
 109     CachedTokens.push_back(Token());
 110     Lex(CachedTokens.back());
 111   }
 112   EnterCachingLexMode();
 113   return CachedTokens.back();
 114 }
 115
 116 void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
 117   assert(Tok.isAnnotation() && "Expected annotation token");
 118   assert(CachedLexPos != 0 && "Expected to have some cached tokens");
 119   assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
 120          && "The annotation should be until the most recent cached token");
 121
 122   // Start from the end of the cached tokens list and look for the token
 123   // that is the beginning of the annotation token.
 124   for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
 125     CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
 126     if (AnnotBegin->getLocation() == Tok.getLocation()) {
 127       assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
 128              "The backtrack pos points inside the annotated tokens!");
 129       // Replace the cached tokens with the single annotation token.
 130       if (i < CachedLexPos)
 131         CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
 132       *AnnotBegin = Tok;
 133       CachedLexPos = i;
 134       return;
 135     }
 136   }
 137 }
 138
 139 bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
 140   // There's currently no cached token...
 141   if (!CachedLexPos)
 142     return false;
 143
 144   const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
 145   if (LastCachedTok.getKind() != Tok.getKind())
 146     return false;
 147
 148   SourceLocation::IntTy RelOffset = 0;
 149   if ((!getSourceManager().isInSameSLocAddrSpace(
 150           Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
 151       RelOffset)
 152     return false;
 153
 154   return true;
 155 }
 156
 157 void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
 158   assert(CachedLexPos != 0 && "Expected to have some cached tokens");
 159   CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
 160                       NewToks.end());
 161   CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
 162   CachedLexPos += NewToks.size() - 1;
 163 }