clang/lib/Format/TokenAnalyzer.cpp

   1 //===--- TokenAnalyzer.cpp - Analyze Token Streams --------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements an abstract TokenAnalyzer and associated helper
  11 /// classes. TokenAnalyzer can be extended to generate replacements based on
  12 /// an annotated and pre-processed token stream.
  13 ///
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "TokenAnalyzer.h"
  17 #include "AffectedRangeManager.h"
  18 #include "Encoding.h"
  19 #include "FormatToken.h"
  20 #include "FormatTokenLexer.h"
  21 #include "TokenAnnotator.h"
  22 #include "UnwrappedLineParser.h"
  23 #include "clang/Basic/Diagnostic.h"
  24 #include "clang/Basic/DiagnosticOptions.h"
  25 #include "clang/Basic/FileManager.h"
  26 #include "clang/Basic/SourceManager.h"
  27 #include "clang/Format/Format.h"
  28 #include "llvm/ADT/STLExtras.h"
  29 #include "llvm/ADT/SmallVector.h"
  30 #include "llvm/Support/Debug.h"
  31 #include <type_traits>
  32
  33 #define DEBUG_TYPE "format-formatter"
  34
  35 namespace clang {
  36 namespace format {
  37
  38 // FIXME: Instead of printing the diagnostic we should store it and have a
  39 // better way to return errors through the format APIs.
  40 class FatalDiagnosticConsumer : public DiagnosticConsumer {
  41 public:
  42   void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
  43                         const Diagnostic &Info) override {
  44     if (DiagLevel == DiagnosticsEngine::Fatal) {
  45       Fatal = true;
  46       llvm::SmallVector<char, 128> Message;
  47       Info.FormatDiagnostic(Message);
  48       llvm::errs() << Message << "\n";
  49     }
  50   }
  51
  52   bool fatalError() const { return Fatal; }
  53
  54 private:
  55   bool Fatal = false;
  56 };
  57
  58 std::unique_ptr<Environment>
  59 Environment::make(StringRef Code, StringRef FileName,
  60                   ArrayRef<tooling::Range> Ranges, unsigned FirstStartColumn,
  61                   unsigned NextStartColumn, unsigned LastStartColumn) {
  62   auto Env = std::make_unique<Environment>(Code, FileName, FirstStartColumn,
  63                                            NextStartColumn, LastStartColumn);
  64   FatalDiagnosticConsumer Diags;
  65   Env->SM.getDiagnostics().setClient(&Diags, /*ShouldOwnClient=*/false);
  66   SourceLocation StartOfFile = Env->SM.getLocForStartOfFile(Env->ID);
  67   for (const tooling::Range &Range : Ranges) {
  68     SourceLocation Start = StartOfFile.getLocWithOffset(Range.getOffset());
  69     SourceLocation End = Start.getLocWithOffset(Range.getLength());
  70     Env->CharRanges.push_back(CharSourceRange::getCharRange(Start, End));
  71   }
  72   // Validate that we can get the buffer data without a fatal error.
  73   Env->SM.getBufferData(Env->ID);
  74   if (Diags.fatalError())
  75     return nullptr;
  76   return Env;
  77 }
  78
  79 Environment::Environment(StringRef Code, StringRef FileName,
  80                          unsigned FirstStartColumn, unsigned NextStartColumn,
  81                          unsigned LastStartColumn)
  82     : VirtualSM(new SourceManagerForFile(FileName, Code)), SM(VirtualSM->get()),
  83       ID(VirtualSM->get().getMainFileID()), FirstStartColumn(FirstStartColumn),
  84       NextStartColumn(NextStartColumn), LastStartColumn(LastStartColumn) {}
  85
  86 TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)
  87     : Style(Style), Env(Env),
  88       AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()),
  89       UnwrappedLines(1),
  90       Encoding(encoding::detectEncoding(
  91           Env.getSourceManager().getBufferData(Env.getFileID()))) {
  92   LLVM_DEBUG(
  93       llvm::dbgs() << "File encoding: "
  94                    << (Encoding == encoding::Encoding_UTF8 ? "UTF8" : "unknown")
  95                    << "\n");
  96   LLVM_DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language)
  97                           << "\n");
  98 }
  99
 100 std::pair<tooling::Replacements, unsigned>
 101 TokenAnalyzer::process(bool SkipAnnotation) {
 102   tooling::Replacements Result;
 103   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
 104   IdentifierTable IdentTable(getFormattingLangOpts(Style));
 105   FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
 106                        Env.getFirstStartColumn(), Style, Encoding, Allocator,
 107
 108                        IdentTable);
 109   ArrayRef<FormatToken *> Toks(Lex.lex());
 110   SmallVector<FormatToken *, 10> Tokens(Toks.begin(), Toks.end());
 111   UnwrappedLineParser Parser(Style, Lex.getKeywords(),
 112                              Env.getFirstStartColumn(), Tokens, *this);
 113   Parser.parse();
 114   assert(UnwrappedLines.back().empty());
 115   unsigned Penalty = 0;
 116   for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE; ++Run) {
 117     const auto &Lines = UnwrappedLines[Run];
 118     LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
 119     SmallVector<AnnotatedLine *, 16> AnnotatedLines;
 120     AnnotatedLines.reserve(Lines.size());
 121
 122     TokenAnnotator Annotator(Style, Lex.getKeywords());
 123     for (const UnwrappedLine &Line : Lines) {
 124       AnnotatedLines.push_back(new AnnotatedLine(Line));
 125       if (!SkipAnnotation)
 126         Annotator.annotate(*AnnotatedLines.back());
 127     }
 128
 129     std::pair<tooling::Replacements, unsigned> RunResult =
 130         analyze(Annotator, AnnotatedLines, Lex);
 131
 132     LLVM_DEBUG({
 133       llvm::dbgs() << "Replacements for run " << Run << ":\n";
 134       for (const tooling::Replacement &Fix : RunResult.first)
 135         llvm::dbgs() << Fix.toString() << "\n";
 136     });
 137     for (AnnotatedLine *Line : AnnotatedLines)
 138       delete Line;
 139
 140     Penalty += RunResult.second;
 141     for (const auto &R : RunResult.first) {
 142       auto Err = Result.add(R);
 143       // FIXME: better error handling here. For now, simply return an empty
 144       // Replacements to indicate failure.
 145       if (Err) {
 146         llvm::errs() << llvm::toString(std::move(Err)) << "\n";
 147         return {tooling::Replacements(), 0};
 148       }
 149     }
 150   }
 151   return {Result, Penalty};
 152 }
 153
 154 void TokenAnalyzer::consumeUnwrappedLine(const UnwrappedLine &TheLine) {
 155   assert(!UnwrappedLines.empty());
 156   UnwrappedLines.back().push_back(TheLine);
 157 }
 158
 159 void TokenAnalyzer::finishRun() {
 160   UnwrappedLines.push_back(SmallVector<UnwrappedLine, 16>());
 161 }
 162
 163 } // end namespace format
 164 } // end namespace clang