1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "tools/gn/parse_tree.h"
9 #include "base/stl_util.h"
10 #include "base/strings/string_number_conversions.h"
11 #include "tools/gn/functions.h"
12 #include "tools/gn/operators.h"
13 #include "tools/gn/scope.h"
14 #include "tools/gn/string_utils.h"
18 std::string
IndentFor(int value
) {
19 return std::string(value
, ' ');
22 bool IsSortRangeSeparator(const ParseNode
* node
, const ParseNode
* prev
) {
23 // If it's a block comment, or has an attached comment with a blank line
24 // before it, then we break the range at this point.
25 return node
->AsBlockComment() != nullptr ||
26 (prev
&& node
->comments() && !node
->comments()->before().empty() &&
27 (node
->GetRange().begin().line_number() >
28 prev
->GetRange().end().line_number() +
29 static_cast<int>(node
->comments()->before().size() + 1)));
32 base::StringPiece
GetStringRepresentation(const ParseNode
* node
) {
33 DCHECK(node
->AsLiteral() || node
->AsIdentifier() || node
->AsAccessor());
34 if (node
->AsLiteral())
35 return node
->AsLiteral()->value().value();
36 else if (node
->AsIdentifier())
37 return node
->AsIdentifier()->value().value();
38 else if (node
->AsAccessor())
39 return node
->AsAccessor()->base().value();
40 return base::StringPiece();
45 Comments::Comments() {
48 Comments::~Comments() {
51 void Comments::ReverseSuffix() {
52 for (int i
= 0, j
= static_cast<int>(suffix_
.size() - 1); i
< j
; ++i
, --j
)
53 std::swap(suffix_
[i
], suffix_
[j
]);
56 ParseNode::ParseNode() {
59 ParseNode::~ParseNode() {
62 const AccessorNode
* ParseNode::AsAccessor() const { return nullptr; }
63 const BinaryOpNode
* ParseNode::AsBinaryOp() const { return nullptr; }
64 const BlockCommentNode
* ParseNode::AsBlockComment() const { return nullptr; }
65 const BlockNode
* ParseNode::AsBlock() const { return nullptr; }
66 const ConditionNode
* ParseNode::AsConditionNode() const { return nullptr; }
67 const EndNode
* ParseNode::AsEnd() const { return nullptr; }
68 const FunctionCallNode
* ParseNode::AsFunctionCall() const { return nullptr; }
69 const IdentifierNode
* ParseNode::AsIdentifier() const { return nullptr; }
70 const ListNode
* ParseNode::AsList() const { return nullptr; }
71 const LiteralNode
* ParseNode::AsLiteral() const { return nullptr; }
72 const UnaryOpNode
* ParseNode::AsUnaryOp() const { return nullptr; }
74 Comments
* ParseNode::comments_mutable() {
76 comments_
.reset(new Comments
);
77 return comments_
.get();
80 void ParseNode::PrintComments(std::ostream
& out
, int indent
) const {
82 std::string ind
= IndentFor(indent
+ 1);
83 for (const auto& token
: comments_
->before())
84 out
<< ind
<< "+BEFORE_COMMENT(\"" << token
.value() << "\")\n";
85 for (const auto& token
: comments_
->suffix())
86 out
<< ind
<< "+SUFFIX_COMMENT(\"" << token
.value() << "\")\n";
87 for (const auto& token
: comments_
->after())
88 out
<< ind
<< "+AFTER_COMMENT(\"" << token
.value() << "\")\n";
92 // AccessorNode ---------------------------------------------------------------
94 AccessorNode::AccessorNode() {
97 AccessorNode::~AccessorNode() {
100 const AccessorNode
* AccessorNode::AsAccessor() const {
104 Value
AccessorNode::Execute(Scope
* scope
, Err
* err
) const {
106 return ExecuteArrayAccess(scope
, err
);
108 return ExecuteScopeAccess(scope
, err
);
113 LocationRange
AccessorNode::GetRange() const {
115 return LocationRange(base_
.location(), index_
->GetRange().end());
117 return LocationRange(base_
.location(), member_
->GetRange().end());
119 return LocationRange();
122 Err
AccessorNode::MakeErrorDescribing(const std::string
& msg
,
123 const std::string
& help
) const {
124 return Err(GetRange(), msg
, help
);
127 void AccessorNode::Print(std::ostream
& out
, int indent
) const {
128 out
<< IndentFor(indent
) << "ACCESSOR\n";
129 PrintComments(out
, indent
);
130 out
<< IndentFor(indent
+ 1) << base_
.value() << "\n";
132 index_
->Print(out
, indent
+ 1);
134 member_
->Print(out
, indent
+ 1);
137 Value
AccessorNode::ExecuteArrayAccess(Scope
* scope
, Err
* err
) const {
138 Value index_value
= index_
->Execute(scope
, err
);
139 if (err
->has_error())
141 if (!index_value
.VerifyTypeIs(Value::INTEGER
, err
))
144 const Value
* base_value
= scope
->GetValue(base_
.value(), true);
146 *err
= MakeErrorDescribing("Undefined identifier.");
149 if (!base_value
->VerifyTypeIs(Value::LIST
, err
))
152 int64 index_int
= index_value
.int_value();
154 *err
= Err(index_
->GetRange(), "Negative array subscript.",
155 "You gave me " + base::Int64ToString(index_int
) + ".");
158 size_t index_sizet
= static_cast<size_t>(index_int
);
159 if (index_sizet
>= base_value
->list_value().size()) {
160 *err
= Err(index_
->GetRange(), "Array subscript out of range.",
161 "You gave me " + base::Int64ToString(index_int
) +
162 " but I was expecting something from 0 to " +
164 static_cast<int64
>(base_value
->list_value().size()) - 1) +
169 // Doing this assumes that there's no way in the language to do anything
170 // between the time the reference is created and the time that the reference
171 // is used. If there is, this will crash! Currently, this is just used for
172 // array accesses where this "shouldn't" happen.
173 return base_value
->list_value()[index_sizet
];
176 Value
AccessorNode::ExecuteScopeAccess(Scope
* scope
, Err
* err
) const {
177 // We jump through some hoops here since ideally a.b will count "b" as
178 // accessed in the given scope. The value "a" might be in some normal nested
179 // scope and we can modify it, but it might also be inherited from the
180 // readonly root scope and we can't do used variable tracking on it. (It's
181 // not legal to const cast it away since the root scope will be in readonly
182 // mode and being accessed from multiple threads without locking.) So this
183 // code handles both cases.
184 const Value
* result
= nullptr;
186 // Look up the value in the scope named by "base_".
187 Value
* mutable_base_value
= scope
->GetMutableValue(base_
.value(), true);
188 if (mutable_base_value
) {
189 // Common case: base value is mutable so we can track variable accesses
190 // for unused value warnings.
191 if (!mutable_base_value
->VerifyTypeIs(Value::SCOPE
, err
))
193 result
= mutable_base_value
->scope_value()->GetValue(
194 member_
->value().value(), true);
196 // Fall back to see if the value is on a read-only scope.
197 const Value
* const_base_value
= scope
->GetValue(base_
.value(), true);
198 if (const_base_value
) {
199 // Read only value, don't try to mark the value access as a "used" one.
200 if (!const_base_value
->VerifyTypeIs(Value::SCOPE
, err
))
203 const_base_value
->scope_value()->GetValue(member_
->value().value());
205 *err
= Err(base_
, "Undefined identifier.");
211 *err
= Err(member_
.get(), "No value named \"" +
212 member_
->value().value() + "\" in scope \"" + base_
.value() + "\"");
218 void AccessorNode::SetNewLocation(int line_number
) {
219 Location old
= base_
.location();
221 Location(old
.file(), line_number
, old
.char_offset(), old
.byte()));
224 // BinaryOpNode ---------------------------------------------------------------
226 BinaryOpNode::BinaryOpNode() {
229 BinaryOpNode::~BinaryOpNode() {
232 const BinaryOpNode
* BinaryOpNode::AsBinaryOp() const {
236 Value
BinaryOpNode::Execute(Scope
* scope
, Err
* err
) const {
237 return ExecuteBinaryOperator(scope
, this, left_
.get(), right_
.get(), err
);
240 LocationRange
BinaryOpNode::GetRange() const {
241 return left_
->GetRange().Union(right_
->GetRange());
244 Err
BinaryOpNode::MakeErrorDescribing(const std::string
& msg
,
245 const std::string
& help
) const {
246 return Err(op_
, msg
, help
);
249 void BinaryOpNode::Print(std::ostream
& out
, int indent
) const {
250 out
<< IndentFor(indent
) << "BINARY(" << op_
.value() << ")\n";
251 PrintComments(out
, indent
);
252 left_
->Print(out
, indent
+ 1);
253 right_
->Print(out
, indent
+ 1);
256 // BlockNode ------------------------------------------------------------------
258 BlockNode::BlockNode() {
261 BlockNode::~BlockNode() {
262 STLDeleteContainerPointers(statements_
.begin(), statements_
.end());
265 const BlockNode
* BlockNode::AsBlock() const {
269 Value
BlockNode::Execute(Scope
* scope
, Err
* err
) const {
270 for (size_t i
= 0; i
< statements_
.size() && !err
->has_error(); i
++) {
271 // Check for trying to execute things with no side effects in a block.
272 const ParseNode
* cur
= statements_
[i
];
273 if (cur
->AsList() || cur
->AsLiteral() || cur
->AsUnaryOp() ||
274 cur
->AsIdentifier()) {
275 *err
= cur
->MakeErrorDescribing(
276 "This statement has no effect.",
277 "Either delete it or do something with the result.");
280 cur
->Execute(scope
, err
);
285 LocationRange
BlockNode::GetRange() const {
286 if (begin_token_
.type() != Token::INVALID
&&
287 end_
->value().type() != Token::INVALID
) {
288 return begin_token_
.range().Union(end_
->value().range());
289 } else if (!statements_
.empty()) {
290 return statements_
[0]->GetRange().Union(
291 statements_
[statements_
.size() - 1]->GetRange());
293 return LocationRange();
296 Err
BlockNode::MakeErrorDescribing(const std::string
& msg
,
297 const std::string
& help
) const {
298 return Err(GetRange(), msg
, help
);
301 void BlockNode::Print(std::ostream
& out
, int indent
) const {
302 out
<< IndentFor(indent
) << "BLOCK\n";
303 PrintComments(out
, indent
);
304 for (const auto& statement
: statements_
)
305 statement
->Print(out
, indent
+ 1);
306 if (end_
&& end_
->comments())
307 end_
->Print(out
, indent
+ 1);
310 // ConditionNode --------------------------------------------------------------
312 ConditionNode::ConditionNode() {
315 ConditionNode::~ConditionNode() {
318 const ConditionNode
* ConditionNode::AsConditionNode() const {
322 Value
ConditionNode::Execute(Scope
* scope
, Err
* err
) const {
323 Value condition_result
= condition_
->Execute(scope
, err
);
324 if (err
->has_error())
326 if (condition_result
.type() != Value::BOOLEAN
) {
327 *err
= condition_
->MakeErrorDescribing(
328 "Condition does not evaluate to a boolean value.",
329 std::string("This is a value of type \"") +
330 Value::DescribeType(condition_result
.type()) +
332 err
->AppendRange(if_token_
.range());
336 if (condition_result
.boolean_value()) {
337 if_true_
->Execute(scope
, err
);
338 } else if (if_false_
) {
339 // The else block is optional.
340 if_false_
->Execute(scope
, err
);
346 LocationRange
ConditionNode::GetRange() const {
348 return if_token_
.range().Union(if_false_
->GetRange());
349 return if_token_
.range().Union(if_true_
->GetRange());
352 Err
ConditionNode::MakeErrorDescribing(const std::string
& msg
,
353 const std::string
& help
) const {
354 return Err(if_token_
, msg
, help
);
357 void ConditionNode::Print(std::ostream
& out
, int indent
) const {
358 out
<< IndentFor(indent
) << "CONDITION\n";
359 PrintComments(out
, indent
);
360 condition_
->Print(out
, indent
+ 1);
361 if_true_
->Print(out
, indent
+ 1);
363 if_false_
->Print(out
, indent
+ 1);
366 // FunctionCallNode -----------------------------------------------------------
368 FunctionCallNode::FunctionCallNode() {
371 FunctionCallNode::~FunctionCallNode() {
374 const FunctionCallNode
* FunctionCallNode::AsFunctionCall() const {
378 Value
FunctionCallNode::Execute(Scope
* scope
, Err
* err
) const {
379 return functions::RunFunction(scope
, this, args_
.get(), block_
.get(), err
);
382 LocationRange
FunctionCallNode::GetRange() const {
383 if (function_
.type() == Token::INVALID
)
384 return LocationRange(); // This will be null in some tests.
386 return function_
.range().Union(block_
->GetRange());
387 return function_
.range().Union(args_
->GetRange());
390 Err
FunctionCallNode::MakeErrorDescribing(const std::string
& msg
,
391 const std::string
& help
) const {
392 return Err(function_
, msg
, help
);
395 void FunctionCallNode::Print(std::ostream
& out
, int indent
) const {
396 out
<< IndentFor(indent
) << "FUNCTION(" << function_
.value() << ")\n";
397 PrintComments(out
, indent
);
398 args_
->Print(out
, indent
+ 1);
400 block_
->Print(out
, indent
+ 1);
403 // IdentifierNode --------------------------------------------------------------
405 IdentifierNode::IdentifierNode() {
408 IdentifierNode::IdentifierNode(const Token
& token
) : value_(token
) {
411 IdentifierNode::~IdentifierNode() {
414 const IdentifierNode
* IdentifierNode::AsIdentifier() const {
418 Value
IdentifierNode::Execute(Scope
* scope
, Err
* err
) const {
419 const Value
* value
= scope
->GetValue(value_
.value(), true);
422 *err
= MakeErrorDescribing("Undefined identifier");
427 result
.set_origin(this);
431 LocationRange
IdentifierNode::GetRange() const {
432 return value_
.range();
435 Err
IdentifierNode::MakeErrorDescribing(const std::string
& msg
,
436 const std::string
& help
) const {
437 return Err(value_
, msg
, help
);
440 void IdentifierNode::Print(std::ostream
& out
, int indent
) const {
441 out
<< IndentFor(indent
) << "IDENTIFIER(" << value_
.value() << ")\n";
442 PrintComments(out
, indent
);
445 void IdentifierNode::SetNewLocation(int line_number
) {
446 Location old
= value_
.location();
448 Location(old
.file(), line_number
, old
.char_offset(), old
.byte()));
451 // ListNode -------------------------------------------------------------------
453 ListNode::ListNode() : prefer_multiline_(false) {
456 ListNode::~ListNode() {
457 STLDeleteContainerPointers(contents_
.begin(), contents_
.end());
460 const ListNode
* ListNode::AsList() const {
464 Value
ListNode::Execute(Scope
* scope
, Err
* err
) const {
465 Value
result_value(this, Value::LIST
);
466 std::vector
<Value
>& results
= result_value
.list_value();
467 results
.reserve(contents_
.size());
469 for (const auto& cur
: contents_
) {
470 if (cur
->AsBlockComment())
472 results
.push_back(cur
->Execute(scope
, err
));
473 if (err
->has_error())
475 if (results
.back().type() == Value::NONE
) {
476 *err
= cur
->MakeErrorDescribing(
477 "This does not evaluate to a value.",
478 "I can't do something with nothing.");
485 LocationRange
ListNode::GetRange() const {
486 return LocationRange(begin_token_
.location(),
487 end_
->value().location());
490 Err
ListNode::MakeErrorDescribing(const std::string
& msg
,
491 const std::string
& help
) const {
492 return Err(begin_token_
, msg
, help
);
495 void ListNode::Print(std::ostream
& out
, int indent
) const {
496 out
<< IndentFor(indent
) << "LIST" << (prefer_multiline_
? " multiline" : "")
498 PrintComments(out
, indent
);
499 for (const auto& cur
: contents_
)
500 cur
->Print(out
, indent
+ 1);
501 if (end_
&& end_
->comments())
502 end_
->Print(out
, indent
+ 1);
505 void ListNode::SortAsStringsList() {
506 // Sorts alphabetically. Partitions first on BlockCommentNodes and sorts each
507 // partition separately.
508 for (auto sr
: GetSortRanges()) {
509 // Save the original line number so that we can re-assign ranges. We assume
510 // they're contiguous lines because GetSortRanges() does so above. We need
511 // to re-assign these line numbers primiarily because `gn format` uses them
512 // to determine whether two nodes were initially separated by a blank line
514 int start_line
= contents_
[sr
.begin
]->GetRange().begin().line_number();
515 const ParseNode
* original_first
= contents_
[sr
.begin
];
516 std::sort(contents_
.begin() + sr
.begin
, contents_
.begin() + sr
.end
,
517 [](const ParseNode
* a
, const ParseNode
* b
) {
518 base::StringPiece astr
= GetStringRepresentation(a
);
519 base::StringPiece bstr
= GetStringRepresentation(b
);
522 // If the beginning of the range had before comments, and the first node
523 // moved during the sort, then move its comments to the new head of the
525 if (original_first
->comments() && contents_
[sr
.begin
] != original_first
) {
526 for (const auto& hc
: original_first
->comments()->before()) {
527 const_cast<ParseNode
*>(contents_
[sr
.begin
])
531 const_cast<ParseNode
*>(original_first
)
535 const ParseNode
* prev
= nullptr;
536 for (size_t i
= sr
.begin
; i
!= sr
.end
; ++i
) {
537 const ParseNode
* node
= contents_
[i
];
538 DCHECK(node
->AsLiteral() || node
->AsIdentifier() || node
->AsAccessor());
540 prev
? prev
->GetRange().end().line_number() + 1 : start_line
;
541 if (node
->AsLiteral()) {
542 const_cast<LiteralNode
*>(node
->AsLiteral())
543 ->SetNewLocation(line_number
);
544 } else if (node
->AsIdentifier()) {
545 const_cast<IdentifierNode
*>(node
->AsIdentifier())
546 ->SetNewLocation(line_number
);
547 } else if (node
->AsAccessor()) {
548 const_cast<AccessorNode
*>(node
->AsAccessor())
549 ->SetNewLocation(line_number
);
556 // Breaks the ParseNodes of |contents| up by ranges that should be separately
557 // sorted. In particular, we break at a block comment, or an item that has an
558 // attached "before" comment and is separated by a blank line from the item
559 // before it. The assumption is that both of these indicate a separate 'section'
560 // of a sources block across which items should not be inter-sorted.
561 std::vector
<ListNode::SortRange
> ListNode::GetSortRanges() const {
562 std::vector
<SortRange
> ranges
;
563 const ParseNode
* prev
= nullptr;
565 for (size_t i
= begin
; i
< contents_
.size(); prev
= contents_
[i
++]) {
566 if (IsSortRangeSeparator(contents_
[i
], prev
)) {
568 ranges
.push_back(SortRange(begin
, i
));
569 // If |i| is an item with an attached comment, then we start the next
570 // range at that point, because we want to include it in the sort.
571 // Otherwise, it's a block comment which we skip over entirely because
572 // we don't want to move or include it in the sort. The two cases are:
579 // # This is a block comment.
586 // which contains 5 elements, and for which the ranges would be { [0,
587 // 2), [3, 5) } (notably excluding 2, the block comment), and:
593 // # This is a header comment.
598 // which contains 4 elements, index 2 containing an attached 'before'
599 // comments, and the ranges should be { [0, 2), [2, 4) }.
600 if (!contents_
[i
]->AsBlockComment())
605 // If it was a one item range, just skip over it.
610 if (begin
!= contents_
.size())
611 ranges
.push_back(SortRange(begin
, contents_
.size()));
615 // LiteralNode -----------------------------------------------------------------
617 LiteralNode::LiteralNode() {
620 LiteralNode::LiteralNode(const Token
& token
) : value_(token
) {
623 LiteralNode::~LiteralNode() {
626 const LiteralNode
* LiteralNode::AsLiteral() const {
630 Value
LiteralNode::Execute(Scope
* scope
, Err
* err
) const {
631 switch (value_
.type()) {
632 case Token::TRUE_TOKEN
:
633 return Value(this, true);
634 case Token::FALSE_TOKEN
:
635 return Value(this, false);
636 case Token::INTEGER
: {
637 base::StringPiece s
= value_
.value();
638 if ((s
.starts_with("0") && s
.size() > 1) || s
.starts_with("-0")) {
640 *err
= MakeErrorDescribing("Negative zero doesn't make sense");
642 *err
= MakeErrorDescribing("Leading zeros not allowed");
646 if (!base::StringToInt64(s
, &result_int
)) {
647 *err
= MakeErrorDescribing("This does not look like an integer");
650 return Value(this, result_int
);
652 case Token::STRING
: {
653 Value
v(this, Value::STRING
);
654 ExpandStringLiteral(scope
, value_
, &v
, err
);
663 LocationRange
LiteralNode::GetRange() const {
664 return value_
.range();
667 Err
LiteralNode::MakeErrorDescribing(const std::string
& msg
,
668 const std::string
& help
) const {
669 return Err(value_
, msg
, help
);
672 void LiteralNode::Print(std::ostream
& out
, int indent
) const {
673 out
<< IndentFor(indent
) << "LITERAL(" << value_
.value() << ")\n";
674 PrintComments(out
, indent
);
677 void LiteralNode::SetNewLocation(int line_number
) {
678 Location old
= value_
.location();
680 Location(old
.file(), line_number
, old
.char_offset(), old
.byte()));
683 // UnaryOpNode ----------------------------------------------------------------
685 UnaryOpNode::UnaryOpNode() {
688 UnaryOpNode::~UnaryOpNode() {
691 const UnaryOpNode
* UnaryOpNode::AsUnaryOp() const {
695 Value
UnaryOpNode::Execute(Scope
* scope
, Err
* err
) const {
696 Value operand_value
= operand_
->Execute(scope
, err
);
697 if (err
->has_error())
699 return ExecuteUnaryOperator(scope
, this, operand_value
, err
);
702 LocationRange
UnaryOpNode::GetRange() const {
703 return op_
.range().Union(operand_
->GetRange());
706 Err
UnaryOpNode::MakeErrorDescribing(const std::string
& msg
,
707 const std::string
& help
) const {
708 return Err(op_
, msg
, help
);
711 void UnaryOpNode::Print(std::ostream
& out
, int indent
) const {
712 out
<< IndentFor(indent
) << "UNARY(" << op_
.value() << ")\n";
713 PrintComments(out
, indent
);
714 operand_
->Print(out
, indent
+ 1);
717 // BlockCommentNode ------------------------------------------------------------
719 BlockCommentNode::BlockCommentNode() {
722 BlockCommentNode::~BlockCommentNode() {
725 const BlockCommentNode
* BlockCommentNode::AsBlockComment() const {
729 Value
BlockCommentNode::Execute(Scope
* scope
, Err
* err
) const {
733 LocationRange
BlockCommentNode::GetRange() const {
734 return comment_
.range();
737 Err
BlockCommentNode::MakeErrorDescribing(const std::string
& msg
,
738 const std::string
& help
) const {
739 return Err(comment_
, msg
, help
);
742 void BlockCommentNode::Print(std::ostream
& out
, int indent
) const {
743 out
<< IndentFor(indent
) << "BLOCK_COMMENT(" << comment_
.value() << ")\n";
744 PrintComments(out
, indent
);
747 // EndNode ---------------------------------------------------------------------
749 EndNode::EndNode(const Token
& token
) : value_(token
) {
752 EndNode::~EndNode() {
755 const EndNode
* EndNode::AsEnd() const {
759 Value
EndNode::Execute(Scope
* scope
, Err
* err
) const {
763 LocationRange
EndNode::GetRange() const {
764 return value_
.range();
767 Err
EndNode::MakeErrorDescribing(const std::string
& msg
,
768 const std::string
& help
) const {
769 return Err(value_
, msg
, help
);
772 void EndNode::Print(std::ostream
& out
, int indent
) const {
773 out
<< IndentFor(indent
) << "END(" << value_
.value() << ")\n";
774 PrintComments(out
, indent
);