clang-tools-extra/clangd/support/Token.cpp

   1 //===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "Token.h"
  10 #include "clang/Basic/LangOptions.h"
  11 #include "llvm/ADT/StringExtras.h"
  12 #include "llvm/Support/Format.h"
  13 #include "llvm/Support/FormatVariadic.h"
  14
  15 namespace clang {
  16 namespace clangd {
  17
  18 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
  19   OS << llvm::formatv("{0} {1}:{2} ", clang::tok::getTokenName(T.Kind), T.Line,
  20                       T.Indent);
  21   OS << '"';
  22   llvm::printEscapedString(T.text(), OS);
  23   OS << '"';
  24   if (T.Flags)
  25     OS << llvm::format(" flags=%x", T.Flags);
  26   return OS;
  27 }
  28
  29 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) {
  30   OS << "Index               Kind    Line  Text\n";
  31   for (const auto &T : TS.tokens()) {
  32     OS << llvm::format("%5d:  %16s %4d:%-2d  ", TS.index(T),
  33                        clang::tok::getTokenName(T.Kind), T.Line, T.Indent);
  34     OS << '"';
  35     llvm::printEscapedString(T.text(), OS);
  36     OS << '"';
  37     if (T.Flags)
  38       OS << llvm::format("  flags=%x", T.Flags);
  39     OS << '\n';
  40   }
  41   return OS;
  42 }
  43
  44 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) {
  45   OS << llvm::formatv("[{0},{1})", R.Begin, R.End);
  46   return OS;
  47 }
  48
  49 TokenStream::TokenStream(std::shared_ptr<void> Payload)
  50     : Payload(std::move(Payload)) {
  51   Storage.emplace_back();
  52   Storage.back().Kind = clang::tok::eof;
  53 }
  54
  55 void TokenStream::finalize() {
  56   assert(!isFinalized());
  57   unsigned LastLine = Storage.back().Line;
  58   Storage.emplace_back();
  59   Storage.back().Kind = tok::eof;
  60   Storage.back().Line = LastLine + 1;
  61
  62   Tokens = Storage;
  63   Tokens = Tokens.drop_front().drop_back();
  64 }
  65
  66 bool TokenStream::isFinalized() const {
  67   assert(!Storage.empty() && Storage.front().Kind == tok::eof);
  68   if (Storage.size() == 1)
  69     return false;
  70   return Storage.back().Kind == tok::eof;
  71 }
  72
  73 void TokenStream::print(llvm::raw_ostream &OS) const {
  74   bool FirstToken = true;
  75   unsigned LastLine = -1;
  76   StringRef LastText;
  77   for (const auto &T : tokens()) {
  78     StringRef Text = T.text();
  79     if (FirstToken) {
  80       FirstToken = false;
  81     } else if (T.Line == LastLine) {
  82       if (LastText.data() + LastText.size() != Text.data())
  83         OS << ' ';
  84     } else {
  85       OS << '\n';
  86       OS.indent(T.Indent);
  87     }
  88     OS << Text;
  89     LastLine = T.Line;
  90     LastText = Text;
  91   }
  92   if (!FirstToken)
  93     OS << '\n';
  94 }
  95
  96 clang::LangOptions genericLangOpts(clang::Language Lang,
  97                                    clang::LangStandard::Kind Standard) {
  98   clang::LangOptions Opts;
  99   std::vector<std::string> UnusedIncludes;
 100   LangOptions::setLangDefaults(Opts, Lang, llvm::Triple(), UnusedIncludes,
 101                                Standard);
 102
 103   // Some options are "on by default", but e.g. at the driver level.
 104   if (Opts.CPlusPlus)
 105     Opts.CXXOperatorNames = true;
 106   if (Opts.CPlusPlus20)
 107     Opts.Coroutines = true;
 108
 109   // Some options are off by default, but define keywords we want to tolerate.
 110   if (Opts.CPlusPlus)
 111     Opts.MicrosoftExt = true;  // kw__try, kw__finally
 112   Opts.DeclSpecKeyword = true; // __declspec
 113   Opts.WChar = true;
 114
 115   return Opts;
 116 }
 117
 118 TokenStream stripComments(const TokenStream &Input) {
 119   TokenStream Out(Input.getPayload());
 120   for (const Token &T : Input.tokens()) {
 121     if (T.Kind == tok::comment)
 122       continue;
 123     Out.push(T);
 124   }
 125   Out.finalize();
 126   return Out;
 127 }
 128
 129 } // namespace clangd
 130 } // namespace clang