1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/performance_monitor/database.h"
7 #include "base/file_util.h"
8 #include "base/files/file_path.h"
9 #include "base/json/json_reader.h"
10 #include "base/json/json_writer.h"
11 #include "base/logging.h"
12 #include "base/path_service.h"
13 #include "base/stl_util.h"
14 #include "base/strings/string_number_conversions.h"
15 #include "base/strings/utf_string_conversions.h"
16 #include "base/time/time.h"
17 #include "chrome/browser/performance_monitor/key_builder.h"
18 #include "chrome/common/chrome_paths.h"
19 #include "content/public/browser/browser_thread.h"
20 #include "third_party/leveldatabase/src/include/leveldb/db.h"
21 #include "third_party/leveldatabase/src/include/leveldb/iterator.h"
22 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
24 namespace performance_monitor
{
26 const char kDbDir
[] = "Performance Monitor Databases";
27 const char kRecentDb
[] = "Recent Metrics";
28 const char kMaxValueDb
[] = "Max Value Metrics";
29 const char kEventDb
[] = "Events";
30 const char kStateDb
[] = "Configuration";
31 const char kActiveIntervalDb
[] = "Active Interval";
32 const char kMetricDb
[] = "Metrics";
33 const double kDefaultMaxValue
= 0.0;
35 // If the db is quiet for this number of minutes, then it is considered down.
36 const base::TimeDelta
kActiveIntervalTimeout() {
37 return base::TimeDelta::FromMinutes(5);
40 TimeRange
ActiveIntervalToTimeRange(const std::string
& start_time
,
41 const std::string
& end_time
) {
42 int64 start_time_int
= 0;
43 int64 end_time_int
= 0;
44 base::StringToInt64(start_time
, &start_time_int
);
45 base::StringToInt64(end_time
, &end_time_int
);
46 return TimeRange(base::Time::FromInternalValue(start_time_int
),
47 base::Time::FromInternalValue(end_time_int
));
50 double StringToDouble(const std::string
& s
) {
52 if (!base::StringToDouble(s
, &value
))
53 LOG(ERROR
) << "Failed to convert " << s
<< " to double.";
57 // Returns an event from the given JSON string; the scoped_ptr will be NULL if
58 // we are unable to properly parse the JSON.
59 scoped_ptr
<Event
> EventFromJSON(const std::string
& data
) {
60 base::Value
* value
= base::JSONReader::Read(data
);
61 base::DictionaryValue
* dict
= NULL
;
62 if (!value
|| !value
->GetAsDictionary(&dict
))
63 return scoped_ptr
<Event
>();
65 return Event::FromValue(scoped_ptr
<base::DictionaryValue
>(dict
));
70 const char Database::kDatabaseSequenceToken
[] =
71 "_performance_monitor_db_sequence_token_";
73 TimeRange::TimeRange() {
76 TimeRange::TimeRange(base::Time start_time
, base::Time end_time
)
81 TimeRange::~TimeRange() {
84 base::Time
Database::SystemClock::GetTime() {
85 return base::Time::Now();
89 scoped_ptr
<Database
> Database::Create(base::FilePath path
) {
90 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
92 CHECK(PathService::Get(chrome::DIR_USER_DATA
, &path
));
93 path
= path
.AppendASCII(kDbDir
);
95 scoped_ptr
<Database
> database
;
96 if (!base::DirectoryExists(path
) && !base::CreateDirectory(path
))
97 return database
.Pass();
98 database
.reset(new Database(path
));
100 // If the database did not initialize correctly, return a NULL scoped_ptr.
101 if (!database
->valid_
)
103 return database
.Pass();
106 bool Database::AddStateValue(const std::string
& key
, const std::string
& value
) {
107 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
108 UpdateActiveInterval();
109 leveldb::Status insert_status
= state_db_
->Put(write_options_
, key
, value
);
110 return insert_status
.ok();
113 std::string
Database::GetStateValue(const std::string
& key
) {
114 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
116 state_db_
->Get(read_options_
, key
, &result
);
120 bool Database::AddEvent(const Event
& event
) {
121 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
122 UpdateActiveInterval();
124 base::JSONWriter::Write(event
.data(), &value
);
125 std::string key
= key_builder_
->CreateEventKey(event
.time(), event
.type());
126 leveldb::Status status
= event_db_
->Put(write_options_
, key
, value
);
130 std::vector
<TimeRange
> Database::GetActiveIntervals(const base::Time
& start
,
131 const base::Time
& end
) {
132 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
133 std::vector
<TimeRange
> results
;
134 std::string start_key
= key_builder_
->CreateActiveIntervalKey(start
);
135 std::string end_key
= key_builder_
->CreateActiveIntervalKey(end
);
136 scoped_ptr
<leveldb::Iterator
> it(active_interval_db_
->NewIterator(
139 // If the interator is valid, we check the previous value in case we jumped
140 // into the middle of an active interval. If the iterator is not valid, then
141 // the key may be in the current active interval.
146 if (it
->Valid() && it
->value().ToString() > start_key
) {
147 results
.push_back(ActiveIntervalToTimeRange(it
->key().ToString(),
148 it
->value().ToString()));
151 for (it
->Seek(start_key
);
152 it
->Valid() && it
->key().ToString() < end_key
;
154 results
.push_back(ActiveIntervalToTimeRange(it
->key().ToString(),
155 it
->value().ToString()));
160 Database::EventVector
Database::GetEvents(EventType type
,
161 const base::Time
& start
,
162 const base::Time
& end
) {
163 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
165 std::string start_key
=
166 key_builder_
->CreateEventKey(start
, EVENT_UNDEFINED
);
167 std::string end_key
=
168 key_builder_
->CreateEventKey(end
, EVENT_NUMBER_OF_EVENTS
);
169 leveldb::WriteBatch invalid_entries
;
170 scoped_ptr
<leveldb::Iterator
> it(event_db_
->NewIterator(read_options_
));
171 for (it
->Seek(start_key
);
172 it
->Valid() && it
->key().ToString() <= end_key
;
174 if (type
!= EVENT_UNDEFINED
) {
176 key_builder_
->EventKeyToEventType(it
->key().ToString());
177 if (key_type
!= type
)
180 scoped_ptr
<Event
> event
= EventFromJSON(it
->value().ToString());
182 invalid_entries
.Delete(it
->key());
183 LOG(ERROR
) << "Found invalid event in the database. JSON: '"
184 << it
->value().ToString()
185 << "'. Erasing event from the database.";
188 events
.push_back(linked_ptr
<Event
>(event
.release()));
190 event_db_
->Write(write_options_
, &invalid_entries
);
194 Database::EventTypeSet
Database::GetEventTypes(const base::Time
& start
,
195 const base::Time
& end
) {
196 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
197 EventTypeSet results
;
198 std::string start_key
=
199 key_builder_
->CreateEventKey(start
, EVENT_UNDEFINED
);
200 std::string end_key
=
201 key_builder_
->CreateEventKey(end
, EVENT_NUMBER_OF_EVENTS
);
202 scoped_ptr
<leveldb::Iterator
> it(event_db_
->NewIterator(read_options_
));
203 for (it
->Seek(start_key
);
204 it
->Valid() && it
->key().ToString() <= end_key
;
207 key_builder_
->EventKeyToEventType(it
->key().ToString());
208 results
.insert(key_type
);
213 bool Database::AddMetric(const std::string
& activity
,
214 const Metric
& metric
) {
215 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
216 if (!metric
.IsValid()) {
217 DLOG(ERROR
) << "Metric to be added is invalid. Type: " << metric
.type
218 << ", Time: " << metric
.time
.ToInternalValue()
219 << ", Value: " << metric
.value
<< ". Ignoring.";
223 UpdateActiveInterval();
224 std::string recent_key
=
225 key_builder_
->CreateRecentKey(metric
.time
, metric
.type
, activity
);
226 std::string metric_key
=
227 key_builder_
->CreateMetricKey(metric
.time
, metric
.type
, activity
);
228 std::string recent_map_key
=
229 key_builder_
->CreateRecentMapKey(metric
.type
, activity
);
230 // Use recent_map_ to quickly find the key that must be removed.
231 RecentMap::iterator old_it
= recent_map_
.find(recent_map_key
);
232 if (old_it
!= recent_map_
.end())
233 recent_db_
->Delete(write_options_
, old_it
->second
);
234 recent_map_
[recent_map_key
] = recent_key
;
235 leveldb::Status recent_status
=
236 recent_db_
->Put(write_options_
, recent_key
, metric
.ValueAsString());
237 leveldb::Status metric_status
=
238 metric_db_
->Put(write_options_
, metric_key
, metric
.ValueAsString());
240 bool max_value_success
=
241 UpdateMaxValue(activity
, metric
.type
, metric
.ValueAsString());
242 return recent_status
.ok() && metric_status
.ok() && max_value_success
;
245 bool Database::UpdateMaxValue(const std::string
& activity
,
247 const std::string
& value
) {
248 std::string
max_value_key(
249 key_builder_
->CreateMaxValueKey(metric
, activity
));
250 bool has_key
= ContainsKey(max_value_map_
, max_value_key
);
251 if ((has_key
&& StringToDouble(value
) > max_value_map_
[max_value_key
]) ||
253 max_value_map_
[max_value_key
] = StringToDouble(value
);
254 return max_value_db_
->Put(write_options_
, max_value_key
, value
).ok();
260 Database::MetricTypeSet
Database::GetActiveMetrics(const base::Time
& start
,
261 const base::Time
& end
) {
262 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
263 std::string recent_start_key
= key_builder_
->CreateRecentKey(
264 start
, static_cast<MetricType
>(0), std::string());
265 std::string recent_end_key
= key_builder_
->CreateRecentKey(
266 end
, METRIC_NUMBER_OF_METRICS
, std::string());
267 std::string recent_end_of_time_key
= key_builder_
->CreateRecentKey(
268 clock_
->GetTime(), METRIC_NUMBER_OF_METRICS
, std::string());
270 MetricTypeSet active_metrics
;
271 // Get all the guaranteed metrics.
272 scoped_ptr
<leveldb::Iterator
> recent_it(
273 recent_db_
->NewIterator(read_options_
));
274 for (recent_it
->Seek(recent_start_key
);
275 recent_it
->Valid() && recent_it
->key().ToString() <= recent_end_key
;
277 RecentKey split_key
=
278 key_builder_
->SplitRecentKey(recent_it
->key().ToString());
279 active_metrics
.insert(split_key
.type
);
281 // Get all the possible metrics (metrics that may have been updated after
283 MetricTypeSet possible_metrics
;
284 for (recent_it
->Seek(recent_end_key
);
285 recent_it
->Valid() &&
286 recent_it
->key().ToString() <= recent_end_of_time_key
;
288 RecentKey split_key
=
289 key_builder_
->SplitRecentKey(recent_it
->key().ToString());
290 possible_metrics
.insert(split_key
.type
);
292 MetricTypeSet::iterator possible_it
;
293 scoped_ptr
<leveldb::Iterator
> metric_it(
294 metric_db_
->NewIterator(read_options_
));
295 for (possible_it
= possible_metrics
.begin();
296 possible_it
!= possible_metrics
.end();
298 std::string metric_start_key
=
299 key_builder_
->CreateMetricKey(start
, *possible_it
,std::string());
300 std::string metric_end_key
=
301 key_builder_
->CreateMetricKey(end
, *possible_it
, std::string());
302 metric_it
->Seek(metric_start_key
);
303 // Stats in the timerange from any activity makes the metric active.
304 if (metric_it
->Valid() && metric_it
->key().ToString() <= metric_end_key
) {
305 active_metrics
.insert(*possible_it
);
309 return active_metrics
;
312 std::set
<std::string
> Database::GetActiveActivities(MetricType metric_type
,
313 const base::Time
& start
) {
314 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
315 std::set
<std::string
> results
;
316 std::string start_key
= key_builder_
->CreateRecentKey(
317 start
, static_cast<MetricType
>(0), std::string());
318 scoped_ptr
<leveldb::Iterator
> it(recent_db_
->NewIterator(read_options_
));
319 for (it
->Seek(start_key
); it
->Valid(); it
->Next()) {
320 RecentKey split_key
=
321 key_builder_
->SplitRecentKey(it
->key().ToString());
322 if (split_key
.type
== metric_type
)
323 results
.insert(split_key
.activity
);
328 double Database::GetMaxStatsForActivityAndMetric(const std::string
& activity
,
330 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
331 std::string
max_value_key(
332 key_builder_
->CreateMaxValueKey(metric
, activity
));
333 if (ContainsKey(max_value_map_
, max_value_key
))
334 return max_value_map_
[max_value_key
];
335 return kDefaultMaxValue
;
338 bool Database::GetRecentStatsForActivityAndMetric(const std::string
& activity
,
339 MetricType metric_type
,
341 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
342 std::string recent_map_key
=
343 key_builder_
->CreateRecentMapKey(metric_type
, activity
);
344 if (!ContainsKey(recent_map_
, recent_map_key
))
346 std::string recent_key
= recent_map_
[recent_map_key
];
349 leveldb::Status status
= recent_db_
->Get(read_options_
, recent_key
, &result
);
351 *metric
= Metric(metric_type
,
352 key_builder_
->SplitRecentKey(recent_key
).time
,
357 scoped_ptr
<Database::MetricVector
> Database::GetStatsForActivityAndMetric(
358 const std::string
& activity
,
359 MetricType metric_type
,
360 const base::Time
& start
,
361 const base::Time
& end
) {
362 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
363 scoped_ptr
<MetricVector
> results(new MetricVector());
364 std::string start_key
=
365 key_builder_
->CreateMetricKey(start
, metric_type
, activity
);
366 std::string end_key
=
367 key_builder_
->CreateMetricKey(end
, metric_type
, activity
);
368 leveldb::WriteBatch invalid_entries
;
369 scoped_ptr
<leveldb::Iterator
> it(metric_db_
->NewIterator(read_options_
));
370 for (it
->Seek(start_key
);
371 it
->Valid() && it
->key().ToString() <= end_key
;
373 MetricKey split_key
=
374 key_builder_
->SplitMetricKey(it
->key().ToString());
375 if (split_key
.activity
== activity
) {
376 Metric
metric(metric_type
, split_key
.time
, it
->value().ToString());
377 if (!metric
.IsValid()) {
378 invalid_entries
.Delete(it
->key());
379 LOG(ERROR
) << "Found bad metric in the database. Type: "
380 << metric
.type
<< ", Time: " << metric
.time
.ToInternalValue()
381 << ", Value: " << metric
.value
382 << ". Erasing metric from database.";
385 results
->push_back(metric
);
388 metric_db_
->Write(write_options_
, &invalid_entries
);
389 return results
.Pass();
392 Database::MetricVectorMap
Database::GetStatsForMetricByActivity(
393 MetricType metric_type
,
394 const base::Time
& start
,
395 const base::Time
& end
) {
396 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
397 MetricVectorMap results
;
398 std::string start_key
=
399 key_builder_
->CreateMetricKey(start
, metric_type
, std::string());
400 std::string end_key
=
401 key_builder_
->CreateMetricKey(end
, metric_type
, std::string());
402 leveldb::WriteBatch invalid_entries
;
403 scoped_ptr
<leveldb::Iterator
> it(metric_db_
->NewIterator(read_options_
));
404 for (it
->Seek(start_key
);
405 it
->Valid() && it
->key().ToString() <= end_key
;
407 MetricKey split_key
= key_builder_
->SplitMetricKey(it
->key().ToString());
408 if (!results
[split_key
.activity
].get()) {
409 results
[split_key
.activity
] =
410 linked_ptr
<MetricVector
>(new MetricVector());
412 Metric
metric(metric_type
, split_key
.time
, it
->value().ToString());
413 if (!metric
.IsValid()) {
414 invalid_entries
.Delete(it
->key());
415 LOG(ERROR
) << "Found bad metric in the database. Type: "
416 << metric
.type
<< ", Time: " << metric
.time
.ToInternalValue()
417 << ", Value: " << metric
.value
418 << ". Erasing metric from database.";
421 results
[split_key
.activity
]->push_back(metric
);
423 metric_db_
->Write(write_options_
, &invalid_entries
);
427 Database::Database(const base::FilePath
& path
)
428 : key_builder_(new KeyBuilder()),
430 read_options_(leveldb::ReadOptions()),
431 write_options_(leveldb::WriteOptions()),
437 clock_
= scoped_ptr
<Clock
>(new SystemClock());
441 Database::~Database() {
444 bool Database::InitDBs() {
445 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
446 leveldb::Options open_options
;
447 open_options
.max_open_files
= 0; // Use minimum.
448 open_options
.create_if_missing
= true;
450 // TODO (rdevlin.cronin): This code is ugly. Fix it.
451 recent_db_
= SafelyOpenDatabase(open_options
,
453 true); // fix if damaged
454 max_value_db_
= SafelyOpenDatabase(open_options
,
456 true); // fix if damaged
457 state_db_
= SafelyOpenDatabase(open_options
,
459 true); // fix if damaged
460 active_interval_db_
= SafelyOpenDatabase(open_options
,
462 true); // fix if damaged
463 metric_db_
= SafelyOpenDatabase(open_options
,
465 true); // fix if damaged
466 event_db_
= SafelyOpenDatabase(open_options
,
468 true); // fix if damaged
469 return recent_db_
&& max_value_db_
&& state_db_
&&
470 active_interval_db_
&& metric_db_
&& event_db_
;
473 scoped_ptr
<leveldb::DB
> Database::SafelyOpenDatabase(
474 const leveldb::Options
& options
,
475 const std::string
& path
,
476 bool fix_if_damaged
) {
477 #if defined(OS_POSIX)
478 std::string name
= path_
.AppendASCII(path
).value();
479 #elif defined(OS_WIN)
480 std::string name
= base::WideToUTF8(path_
.AppendASCII(path
).value());
483 leveldb::DB
* database
;
484 leveldb::Status status
= leveldb::DB::Open(options
, name
, &database
);
485 // If all goes well, return the database.
487 return scoped_ptr
<leveldb::DB
>(database
);
489 // Return NULL and print the error if we either didn't find the database and
490 // don't want to create it, or if we don't want to try to fix it.
491 if ((status
.IsNotFound() && !options
.create_if_missing
) || !fix_if_damaged
) {
492 LOG(ERROR
) << status
.ToString();
493 return scoped_ptr
<leveldb::DB
>();
495 // Otherwise, we have an error (corruption, io error, or a not found error
496 // even if we tried to create it).
498 // First, we try again.
499 LOG(ERROR
) << "Database error: " << status
.ToString() << ". Trying again.";
500 status
= leveldb::DB::Open(options
, name
, &database
);
501 // If we fail on corruption, we can try to repair it.
502 if (status
.IsCorruption()) {
503 LOG(ERROR
) << "Database corrupt (second attempt). Trying to repair.";
504 status
= leveldb::RepairDB(name
, options
);
505 // If the repair succeeds and we can open the database, return the
506 // database. Otherwise, continue on.
508 status
= leveldb::DB::Open(options
, name
, &database
);
510 return scoped_ptr
<leveldb::DB
>(database
);
512 LOG(ERROR
) << "Repair failed. Deleting database.";
514 // Next, try to delete and recreate the database. Return NULL if we fail
515 // on either of these steps.
516 status
= leveldb::DestroyDB(name
, options
);
518 LOG(ERROR
) << "Failed to delete database. " << status
.ToString();
519 return scoped_ptr
<leveldb::DB
>();
521 // If we don't have the create_if_missing option, add it (it's safe to
522 // assume this is okay, since we have permission to |fix_if_damaged|).
523 if (!options
.create_if_missing
) {
524 leveldb::Options
create_options(options
);
525 create_options
.create_if_missing
= true;
526 status
= leveldb::DB::Open(create_options
, name
, &database
);
528 status
= leveldb::DB::Open(options
, name
, &database
);
530 // There's nothing else we can try at this point.
532 return scoped_ptr
<leveldb::DB
>(database
);
533 // Return the database if we succeeded, or NULL on failure.
534 LOG(ERROR
) << "Failed to recreate database. " << status
.ToString();
535 return scoped_ptr
<leveldb::DB
>();
538 bool Database::Close() {
539 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
543 max_value_db_
.reset();
545 active_interval_db_
.reset();
546 start_time_key_
.clear();
550 void Database::LoadRecents() {
551 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
553 scoped_ptr
<leveldb::Iterator
> it(recent_db_
->NewIterator(read_options_
));
554 for (it
->SeekToFirst(); it
->Valid(); it
->Next()) {
555 RecentKey split_key
= key_builder_
->SplitRecentKey(it
->key().ToString());
556 recent_map_
[key_builder_
->
557 CreateRecentMapKey(split_key
.type
, split_key
.activity
)] =
558 it
->key().ToString();
562 void Database::LoadMaxValues() {
563 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
564 max_value_map_
.clear();
565 scoped_ptr
<leveldb::Iterator
> it(max_value_db_
->NewIterator(read_options_
));
566 for (it
->SeekToFirst(); it
->Valid(); it
->Next()) {
567 max_value_map_
[it
->key().ToString()] =
568 StringToDouble(it
->value().ToString());
572 // TODO(chebert): Only update the active interval under certian circumstances
573 // eg. every 10 times or when forced.
574 void Database::UpdateActiveInterval() {
575 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI
));
576 base::Time current_time
= clock_
->GetTime();
577 std::string end_time
;
578 // If the last update was too long ago.
579 if (start_time_key_
.empty() ||
580 current_time
- last_update_time_
> kActiveIntervalTimeout()) {
581 start_time_key_
= key_builder_
->CreateActiveIntervalKey(current_time
);
582 end_time
= start_time_key_
;
584 end_time
= key_builder_
->CreateActiveIntervalKey(clock_
->GetTime());
586 last_update_time_
= current_time
;
587 active_interval_db_
->Put(write_options_
, start_time_key_
, end_time
);
590 } // namespace performance_monitor