1 //===-- QualityTests.cpp ----------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // Evaluating scoring functions isn't a great fit for assert-based tests.
10 // For interesting cases, both exact scores and "X beats Y" are too brittle to
11 // make good hard assertions.
13 // Here we test the signal extraction and sanity-check that signals point in
14 // the right direction. This should be supplemented by quality metrics which
15 // we can compute from a corpus of queries and preferred rankings.
17 //===----------------------------------------------------------------------===//
19 #include "FileDistance.h"
23 #include "index/FileIndex.h"
24 #include "clang/AST/Decl.h"
25 #include "clang/AST/DeclCXX.h"
26 #include "clang/Sema/CodeCompleteConsumer.h"
27 #include "llvm/Support/Casting.h"
28 #include "gmock/gmock.h"
29 #include "gtest/gtest.h"
35 // Force the unittest URI scheme to be linked,
36 static int LLVM_ATTRIBUTE_UNUSED UnittestSchemeAnchorDest
=
37 UnittestSchemeAnchorSource
;
41 TEST(QualityTests
, SymbolQualitySignalExtraction
) {
42 auto Header
= TestTU::withHeaderCode(R
"cpp(
46 int _f() { return _X; }
48 #define DECL_NAME(x, y) x##_##y##_Decl
49 #define DECL(x, y) class DECL_NAME(x, y) {};
50 DECL(X, Y); // X_Y_Decl
53 auto Symbols
= Header
.headerSymbols();
54 auto AST
= Header
.build();
56 SymbolQualitySignals Quality
;
57 Quality
.merge(findSymbol(Symbols
, "X_Y_Decl"));
58 EXPECT_TRUE(Quality
.ImplementationDetail
);
60 Symbol F
= findSymbol(Symbols
, "_f");
61 F
.References
= 24; // TestTU doesn't count references, so fake it.
64 EXPECT_TRUE(Quality
.Deprecated
);
65 EXPECT_FALSE(Quality
.ReservedName
);
66 EXPECT_EQ(Quality
.References
, 24u);
67 EXPECT_EQ(Quality
.Category
, SymbolQualitySignals::Function
);
70 Quality
.merge(CodeCompletionResult(&findDecl(AST
, "_f"), /*Priority=*/42));
71 EXPECT_TRUE(Quality
.Deprecated
);
72 EXPECT_FALSE(Quality
.ReservedName
);
73 EXPECT_EQ(Quality
.References
, SymbolQualitySignals().References
);
74 EXPECT_EQ(Quality
.Category
, SymbolQualitySignals::Function
);
77 Quality
.merge(CodeCompletionResult("if"));
78 EXPECT_EQ(Quality
.Category
, SymbolQualitySignals::Keyword
);
80 // Testing ReservedName in main file, we don't index those symbols in headers.
81 auto MainAST
= TestTU::withCode("int _X;").build();
82 SymbolSlab MainSymbols
= std::get
<0>(indexMainDecls(MainAST
));
85 Quality
.merge(findSymbol(MainSymbols
, "_X"));
86 EXPECT_FALSE(Quality
.Deprecated
);
87 EXPECT_FALSE(Quality
.ImplementationDetail
);
88 EXPECT_TRUE(Quality
.ReservedName
);
91 TEST(QualityTests
, SymbolRelevanceSignalExtraction
) {
93 Test
.HeaderCode
= R
"cpp(
97 namespace hdr { class Bar {}; } // namespace hdr
99 #define DEFINE_FLAG(X) \
109 using flags::FLAGS_FOO;
111 int ::header_main() {}
115 int deprecated() { return 0; }
117 namespace { struct X { void y() { int z; } }; }
120 auto AST
= Test
.build();
122 SymbolRelevanceSignals Relevance
;
123 Relevance
.merge(CodeCompletionResult(&findDecl(AST
, "deprecated"),
124 /*Priority=*/42, nullptr, false,
125 /*Accessible=*/false));
126 EXPECT_EQ(Relevance
.NameMatch
, SymbolRelevanceSignals().NameMatch
);
127 EXPECT_TRUE(Relevance
.Forbidden
);
128 EXPECT_EQ(Relevance
.Scope
, SymbolRelevanceSignals::GlobalScope
);
131 Relevance
.merge(CodeCompletionResult(&findDecl(AST
, "main"), 42));
132 EXPECT_FLOAT_EQ(Relevance
.SemaFileProximityScore
, 1.0f
)
133 << "Decl in current file";
135 Relevance
.merge(CodeCompletionResult(&findDecl(AST
, "header"), 42));
136 EXPECT_FLOAT_EQ(Relevance
.SemaFileProximityScore
, 0.6f
) << "Decl from header";
138 Relevance
.merge(CodeCompletionResult(&findDecl(AST
, "header_main"), 42));
139 EXPECT_FLOAT_EQ(Relevance
.SemaFileProximityScore
, 1.0f
)
140 << "Current file and header";
142 auto ConstructShadowDeclCompletionResult
= [&](const std::string DeclName
) {
144 *dyn_cast
<UsingDecl
>(&findDecl(AST
, [&](const NamedDecl
&ND
) {
145 if (const UsingDecl
*Using
= dyn_cast
<UsingDecl
>(&ND
))
146 if (Using
->shadow_size() &&
147 Using
->getQualifiedNameAsString() == DeclName
)
151 CodeCompletionResult
Result(Shadow
->getTargetDecl(), 42);
152 Result
.ShadowDecl
= Shadow
;
157 Relevance
.merge(ConstructShadowDeclCompletionResult("Bar"));
158 EXPECT_FLOAT_EQ(Relevance
.SemaFileProximityScore
, 1.0f
)
159 << "Using declaration in main file";
160 Relevance
.merge(ConstructShadowDeclCompletionResult("FLAGS_FOO"));
161 EXPECT_FLOAT_EQ(Relevance
.SemaFileProximityScore
, 1.0f
)
162 << "Using declaration in main file";
165 Relevance
.merge(CodeCompletionResult(&findUnqualifiedDecl(AST
, "X"), 42));
166 EXPECT_EQ(Relevance
.Scope
, SymbolRelevanceSignals::FileScope
);
168 Relevance
.merge(CodeCompletionResult(&findUnqualifiedDecl(AST
, "y"), 42));
169 EXPECT_EQ(Relevance
.Scope
, SymbolRelevanceSignals::ClassScope
);
171 Relevance
.merge(CodeCompletionResult(&findUnqualifiedDecl(AST
, "z"), 42));
172 EXPECT_EQ(Relevance
.Scope
, SymbolRelevanceSignals::FunctionScope
);
173 // The injected class name is treated as the outer class name.
175 Relevance
.merge(CodeCompletionResult(&findDecl(AST
, "S::S"), 42));
176 EXPECT_EQ(Relevance
.Scope
, SymbolRelevanceSignals::GlobalScope
);
179 EXPECT_FALSE(Relevance
.InBaseClass
);
180 auto BaseMember
= CodeCompletionResult(&findUnqualifiedDecl(AST
, "y"), 42);
181 BaseMember
.InBaseClass
= true;
182 Relevance
.merge(BaseMember
);
183 EXPECT_TRUE(Relevance
.InBaseClass
);
185 auto Index
= Test
.index();
186 FuzzyFindRequest Req
;
189 bool Matched
= false;
190 Index
->fuzzyFind(Req
, [&](const Symbol
&S
) {
194 EXPECT_EQ(Relevance
.Scope
, SymbolRelevanceSignals::FileScope
);
196 EXPECT_TRUE(Matched
);
199 // Do the signals move the scores in the direction we expect?
200 TEST(QualityTests
, SymbolQualitySignalsSanity
) {
201 SymbolQualitySignals Default
;
202 EXPECT_EQ(Default
.evaluateHeuristics(), 1);
204 SymbolQualitySignals Deprecated
;
205 Deprecated
.Deprecated
= true;
206 EXPECT_LT(Deprecated
.evaluateHeuristics(), Default
.evaluateHeuristics());
208 SymbolQualitySignals ReservedName
;
209 ReservedName
.ReservedName
= true;
210 EXPECT_LT(ReservedName
.evaluateHeuristics(), Default
.evaluateHeuristics());
212 SymbolQualitySignals ImplementationDetail
;
213 ImplementationDetail
.ImplementationDetail
= true;
214 EXPECT_LT(ImplementationDetail
.evaluateHeuristics(),
215 Default
.evaluateHeuristics());
217 SymbolQualitySignals WithReferences
, ManyReferences
;
218 WithReferences
.References
= 20;
219 ManyReferences
.References
= 1000;
220 EXPECT_GT(WithReferences
.evaluateHeuristics(), Default
.evaluateHeuristics());
221 EXPECT_GT(ManyReferences
.evaluateHeuristics(),
222 WithReferences
.evaluateHeuristics());
224 SymbolQualitySignals Keyword
, Variable
, Macro
, Constructor
, Function
,
225 Destructor
, Operator
;
226 Keyword
.Category
= SymbolQualitySignals::Keyword
;
227 Variable
.Category
= SymbolQualitySignals::Variable
;
228 Macro
.Category
= SymbolQualitySignals::Macro
;
229 Constructor
.Category
= SymbolQualitySignals::Constructor
;
230 Destructor
.Category
= SymbolQualitySignals::Destructor
;
231 Destructor
.Category
= SymbolQualitySignals::Destructor
;
232 Operator
.Category
= SymbolQualitySignals::Operator
;
233 Function
.Category
= SymbolQualitySignals::Function
;
234 EXPECT_GT(Variable
.evaluateHeuristics(), Default
.evaluateHeuristics());
235 EXPECT_GT(Keyword
.evaluateHeuristics(), Variable
.evaluateHeuristics());
236 EXPECT_LT(Macro
.evaluateHeuristics(), Default
.evaluateHeuristics());
237 EXPECT_LT(Operator
.evaluateHeuristics(), Default
.evaluateHeuristics());
238 EXPECT_LT(Constructor
.evaluateHeuristics(), Function
.evaluateHeuristics());
239 EXPECT_LT(Destructor
.evaluateHeuristics(), Constructor
.evaluateHeuristics());
242 TEST(QualityTests
, SymbolRelevanceSignalsSanity
) {
243 SymbolRelevanceSignals Default
;
244 EXPECT_EQ(Default
.evaluateHeuristics(), 1);
246 SymbolRelevanceSignals Forbidden
;
247 Forbidden
.Forbidden
= true;
248 EXPECT_LT(Forbidden
.evaluateHeuristics(), Default
.evaluateHeuristics());
250 SymbolRelevanceSignals PoorNameMatch
;
251 PoorNameMatch
.NameMatch
= 0.2f
;
252 EXPECT_LT(PoorNameMatch
.evaluateHeuristics(), Default
.evaluateHeuristics());
254 SymbolRelevanceSignals WithSemaFileProximity
;
255 WithSemaFileProximity
.SemaFileProximityScore
= 0.2f
;
256 EXPECT_GT(WithSemaFileProximity
.evaluateHeuristics(),
257 Default
.evaluateHeuristics());
259 ScopeDistance
ScopeProximity({"x::y::"});
261 SymbolRelevanceSignals WithSemaScopeProximity
;
262 WithSemaScopeProximity
.ScopeProximityMatch
= &ScopeProximity
;
263 WithSemaScopeProximity
.SemaSaysInScope
= true;
264 EXPECT_GT(WithSemaScopeProximity
.evaluateHeuristics(),
265 Default
.evaluateHeuristics());
267 SymbolRelevanceSignals WithIndexScopeProximity
;
268 WithIndexScopeProximity
.ScopeProximityMatch
= &ScopeProximity
;
269 WithIndexScopeProximity
.SymbolScope
= "x::";
270 EXPECT_GT(WithSemaScopeProximity
.evaluateHeuristics(),
271 Default
.evaluateHeuristics());
273 SymbolRelevanceSignals IndexProximate
;
274 IndexProximate
.SymbolURI
= "unittest:/foo/bar.h";
275 llvm::StringMap
<SourceParams
> ProxSources
;
276 ProxSources
.try_emplace(testPath("foo/baz.h"));
277 URIDistance
Distance(ProxSources
);
278 IndexProximate
.FileProximityMatch
= &Distance
;
279 EXPECT_GT(IndexProximate
.evaluateHeuristics(), Default
.evaluateHeuristics());
280 SymbolRelevanceSignals IndexDistant
= IndexProximate
;
281 IndexDistant
.SymbolURI
= "unittest:/elsewhere/path.h";
282 EXPECT_GT(IndexProximate
.evaluateHeuristics(),
283 IndexDistant
.evaluateHeuristics())
284 << IndexProximate
<< IndexDistant
;
285 EXPECT_GT(IndexDistant
.evaluateHeuristics(), Default
.evaluateHeuristics());
287 SymbolRelevanceSignals Scoped
;
288 Scoped
.Scope
= SymbolRelevanceSignals::FileScope
;
289 EXPECT_LT(Scoped
.evaluateHeuristics(), Default
.evaluateHeuristics());
290 Scoped
.Query
= SymbolRelevanceSignals::CodeComplete
;
291 EXPECT_GT(Scoped
.evaluateHeuristics(), Default
.evaluateHeuristics());
293 SymbolRelevanceSignals Instance
;
294 Instance
.IsInstanceMember
= false;
295 EXPECT_EQ(Instance
.evaluateHeuristics(), Default
.evaluateHeuristics());
296 Instance
.Context
= CodeCompletionContext::CCC_DotMemberAccess
;
297 EXPECT_LT(Instance
.evaluateHeuristics(), Default
.evaluateHeuristics());
298 Instance
.IsInstanceMember
= true;
299 EXPECT_EQ(Instance
.evaluateHeuristics(), Default
.evaluateHeuristics());
301 SymbolRelevanceSignals InBaseClass
;
302 InBaseClass
.InBaseClass
= true;
303 EXPECT_LT(InBaseClass
.evaluateHeuristics(), Default
.evaluateHeuristics());
305 llvm::StringSet
<> Words
= {"one", "two", "three"};
306 SymbolRelevanceSignals WithoutMatchingWord
;
307 WithoutMatchingWord
.ContextWords
= &Words
;
308 WithoutMatchingWord
.Name
= "four";
309 EXPECT_EQ(WithoutMatchingWord
.evaluateHeuristics(),
310 Default
.evaluateHeuristics());
311 SymbolRelevanceSignals WithMatchingWord
;
312 WithMatchingWord
.ContextWords
= &Words
;
313 WithMatchingWord
.Name
= "TheTwoTowers";
314 EXPECT_GT(WithMatchingWord
.evaluateHeuristics(),
315 Default
.evaluateHeuristics());
318 TEST(QualityTests
, ScopeProximity
) {
319 SymbolRelevanceSignals Relevance
;
320 ScopeDistance
ScopeProximity({"x::y::z::", "x::", "llvm::", ""});
321 Relevance
.ScopeProximityMatch
= &ScopeProximity
;
323 Relevance
.SymbolScope
= "other::";
324 float NotMatched
= Relevance
.evaluateHeuristics();
326 Relevance
.SymbolScope
= "";
327 float Global
= Relevance
.evaluateHeuristics();
328 EXPECT_GT(Global
, NotMatched
);
330 Relevance
.SymbolScope
= "llvm::";
331 float NonParent
= Relevance
.evaluateHeuristics();
332 EXPECT_GT(NonParent
, Global
);
334 Relevance
.SymbolScope
= "x::";
335 float GrandParent
= Relevance
.evaluateHeuristics();
336 EXPECT_GT(GrandParent
, Global
);
338 Relevance
.SymbolScope
= "x::y::";
339 float Parent
= Relevance
.evaluateHeuristics();
340 EXPECT_GT(Parent
, GrandParent
);
342 Relevance
.SymbolScope
= "x::y::z::";
343 float Enclosing
= Relevance
.evaluateHeuristics();
344 EXPECT_GT(Enclosing
, Parent
);
347 TEST(QualityTests
, SortText
) {
348 EXPECT_LT(sortText(std::numeric_limits
<float>::infinity()),
350 EXPECT_LT(sortText(1000.2f
), sortText(1));
351 EXPECT_LT(sortText(1), sortText(0.3f
));
352 EXPECT_LT(sortText(0.3f
), sortText(0));
353 EXPECT_LT(sortText(0), sortText(-10));
354 EXPECT_LT(sortText(-10), sortText(-std::numeric_limits
<float>::infinity()));
356 EXPECT_LT(sortText(1, "z"), sortText(0, "a"));
357 EXPECT_LT(sortText(0, "a"), sortText(0, "z"));
360 TEST(QualityTests
, NoBoostForClassConstructor
) {
361 auto Header
= TestTU::withHeaderCode(R
"cpp(
367 auto Symbols
= Header
.headerSymbols();
368 auto AST
= Header
.build();
370 const NamedDecl
*Foo
= &findDecl(AST
, "Foo");
371 SymbolRelevanceSignals Cls
;
372 Cls
.merge(CodeCompletionResult(Foo
, /*Priority=*/0));
374 const NamedDecl
*CtorDecl
= &findDecl(AST
, [](const NamedDecl
&ND
) {
375 return (ND
.getQualifiedNameAsString() == "Foo::Foo") &&
376 isa
<CXXConstructorDecl
>(&ND
);
378 SymbolRelevanceSignals Ctor
;
379 Ctor
.merge(CodeCompletionResult(CtorDecl
, /*Priority=*/0));
381 EXPECT_EQ(Cls
.Scope
, SymbolRelevanceSignals::GlobalScope
);
382 EXPECT_EQ(Ctor
.Scope
, SymbolRelevanceSignals::GlobalScope
);
385 TEST(QualityTests
, IsInstanceMember
) {
386 auto Header
= TestTU::withHeaderCode(R
"cpp(
391 template <typename T> void tpl(T *t) {}
396 auto Symbols
= Header
.headerSymbols();
398 SymbolRelevanceSignals Rel
;
399 const Symbol
&FooSym
= findSymbol(Symbols
, "Foo::foo");
401 EXPECT_FALSE(Rel
.IsInstanceMember
);
402 const Symbol
&BarSym
= findSymbol(Symbols
, "Foo::bar");
404 EXPECT_TRUE(Rel
.IsInstanceMember
);
406 Rel
.IsInstanceMember
= false;
407 const Symbol
&TplSym
= findSymbol(Symbols
, "Foo::tpl");
409 EXPECT_TRUE(Rel
.IsInstanceMember
);
411 auto AST
= Header
.build();
412 const NamedDecl
*Foo
= &findDecl(AST
, "Foo::foo");
413 const NamedDecl
*Bar
= &findDecl(AST
, "Foo::bar");
414 const NamedDecl
*Tpl
= &findDecl(AST
, "Foo::tpl");
416 Rel
.IsInstanceMember
= false;
417 Rel
.merge(CodeCompletionResult(Foo
, /*Priority=*/0));
418 EXPECT_FALSE(Rel
.IsInstanceMember
);
419 Rel
.merge(CodeCompletionResult(Bar
, /*Priority=*/0));
420 EXPECT_TRUE(Rel
.IsInstanceMember
);
421 Rel
.IsInstanceMember
= false;
422 Rel
.merge(CodeCompletionResult(Tpl
, /*Priority=*/0));
423 EXPECT_TRUE(Rel
.IsInstanceMember
);
426 TEST(QualityTests
, ConstructorDestructor
) {
427 auto Header
= TestTU::withHeaderCode(R
"cpp(
434 auto Symbols
= Header
.headerSymbols();
435 auto AST
= Header
.build();
437 const NamedDecl
*CtorDecl
= &findDecl(AST
, [](const NamedDecl
&ND
) {
438 return (ND
.getQualifiedNameAsString() == "Foo::Foo") &&
439 isa
<CXXConstructorDecl
>(&ND
);
441 const NamedDecl
*DtorDecl
= &findDecl(AST
, [](const NamedDecl
&ND
) {
442 return (ND
.getQualifiedNameAsString() == "Foo::~Foo") &&
443 isa
<CXXDestructorDecl
>(&ND
);
446 SymbolQualitySignals CtorQ
;
447 CtorQ
.merge(CodeCompletionResult(CtorDecl
, /*Priority=*/0));
448 EXPECT_EQ(CtorQ
.Category
, SymbolQualitySignals::Constructor
);
450 CtorQ
.Category
= SymbolQualitySignals::Unknown
;
451 const Symbol
&CtorSym
= findSymbol(Symbols
, "Foo::Foo");
452 CtorQ
.merge(CtorSym
);
453 EXPECT_EQ(CtorQ
.Category
, SymbolQualitySignals::Constructor
);
455 SymbolQualitySignals DtorQ
;
456 DtorQ
.merge(CodeCompletionResult(DtorDecl
, /*Priority=*/0));
457 EXPECT_EQ(DtorQ
.Category
, SymbolQualitySignals::Destructor
);
460 TEST(QualityTests
, Operator
) {
461 auto Header
= TestTU::withHeaderCode(R
"cpp(
464 bool operator<(const Foo& f1);
467 auto AST
= Header
.build();
469 const NamedDecl
*Operator
= &findDecl(AST
, [](const NamedDecl
&ND
) {
470 if (const auto *OD
= dyn_cast
<FunctionDecl
>(&ND
))
471 if (OD
->isOverloadedOperator())
475 SymbolQualitySignals Q
;
476 Q
.merge(CodeCompletionResult(Operator
, /*Priority=*/0));
477 EXPECT_EQ(Q
.Category
, SymbolQualitySignals::Operator
);
480 TEST(QualityTests
, ItemWithFixItsRankedDown
) {
481 CodeCompleteOptions Opts
;
482 Opts
.IncludeFixIts
= true;
484 auto Header
= TestTU::withHeaderCode(R
"cpp(
487 auto AST
= Header
.build();
489 SymbolRelevanceSignals RelevanceWithFixIt
;
490 RelevanceWithFixIt
.merge(CodeCompletionResult(&findDecl(AST
, "x"), 0, nullptr,
491 false, true, {FixItHint
{}}));
492 EXPECT_TRUE(RelevanceWithFixIt
.NeedsFixIts
);
494 SymbolRelevanceSignals RelevanceWithoutFixIt
;
495 RelevanceWithoutFixIt
.merge(
496 CodeCompletionResult(&findDecl(AST
, "x"), 0, nullptr, false, true, {}));
497 EXPECT_FALSE(RelevanceWithoutFixIt
.NeedsFixIts
);
499 EXPECT_LT(RelevanceWithFixIt
.evaluateHeuristics(),
500 RelevanceWithoutFixIt
.evaluateHeuristics());
504 } // namespace clangd