1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/media/webrtc_log_uploader.h"
7 #include "base/files/file_enumerator.h"
8 #include "base/files/file_path.h"
9 #include "base/files/file_util.h"
10 #include "base/logging.h"
11 #include "base/pickle.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/strings/string_split.h"
14 #include "base/strings/stringprintf.h"
15 #include "base/time/time.h"
16 #include "chrome/browser/browser_process.h"
17 #include "chrome/browser/media/media_url_constants.h"
18 #include "chrome/browser/media/webrtc_log_list.h"
19 #include "chrome/browser/media/webrtc_log_util.h"
20 #include "chrome/common/partial_circular_buffer.h"
21 #include "components/version_info/version_info.h"
22 #include "content/public/browser/browser_thread.h"
23 #include "net/base/mime_util.h"
24 #include "net/url_request/url_fetcher.h"
25 #include "third_party/zlib/zlib.h"
29 const int kLogCountLimit
= 5;
30 const uint32 kIntermediateCompressionBufferBytes
= 256 * 1024; // 256 KB
31 const int kLogListLimitLines
= 50;
33 const char kUploadContentType
[] = "multipart/form-data";
34 const char kMultipartBoundary
[] =
35 "----**--yradnuoBgoLtrapitluMklaTelgooG--**----";
37 const int kHttpResponseOk
= 200;
39 // Adds the header section for a gzip file to the multipart |post_data|.
40 void AddMultipartFileContentHeader(std::string
* post_data
,
41 const std::string
& content_name
) {
42 post_data
->append("--");
43 post_data
->append(kMultipartBoundary
);
44 post_data
->append("\r\nContent-Disposition: form-data; name=\"");
45 post_data
->append(content_name
);
46 post_data
->append("\"; filename=\"");
47 post_data
->append(content_name
+ ".gz");
48 post_data
->append("\"\r\nContent-Type: application/gzip\r\n\r\n");
51 // Adds |compressed_log| to |post_data|.
52 void AddLogData(std::string
* post_data
,
53 const std::string
& compressed_log
) {
54 AddMultipartFileContentHeader(post_data
, "webrtc_log");
55 post_data
->append(compressed_log
);
56 post_data
->append("\r\n");
59 // Adds the RTP dump data to |post_data|.
60 void AddRtpDumpData(std::string
* post_data
,
61 const std::string
& name
,
62 const std::string
& dump_data
) {
63 AddMultipartFileContentHeader(post_data
, name
);
64 post_data
->append(dump_data
.data(), dump_data
.size());
65 post_data
->append("\r\n");
70 WebRtcLogUploadDoneData::WebRtcLogUploadDoneData() {}
72 WebRtcLogUploadDoneData::~WebRtcLogUploadDoneData() {}
74 WebRtcLogUploader::WebRtcLogUploader()
77 shutting_down_(false) {
78 file_thread_checker_
.DetachFromThread();
81 WebRtcLogUploader::~WebRtcLogUploader() {
82 DCHECK(create_thread_checker_
.CalledOnValidThread());
83 DCHECK(upload_done_data_
.empty());
84 DCHECK(shutting_down_
);
87 void WebRtcLogUploader::OnURLFetchComplete(
88 const net::URLFetcher
* source
) {
89 DCHECK(create_thread_checker_
.CalledOnValidThread());
90 DCHECK(upload_done_data_
.find(source
) != upload_done_data_
.end());
91 DCHECK(!shutting_down_
);
92 int response_code
= source
->GetResponseCode();
93 UploadDoneDataMap::iterator it
= upload_done_data_
.find(source
);
94 if (it
!= upload_done_data_
.end()) {
95 // The log path can be empty here if we failed getting it before. We still
96 // upload the log if that's the case.
97 std::string report_id
;
98 if (response_code
== kHttpResponseOk
&&
99 source
->GetResponseAsString(&report_id
) &&
100 !it
->second
.log_path
.empty()) {
101 // TODO(jiayl): Add the RTP dump records to chrome://webrtc-logs.
102 base::FilePath log_list_path
=
103 WebRtcLogList::GetWebRtcLogListFileForDirectory(it
->second
.log_path
);
104 content::BrowserThread::PostTask(
105 content::BrowserThread::FILE,
107 base::Bind(&WebRtcLogUploader::AddUploadedLogInfoToUploadListFile
,
108 base::Unretained(this),
110 it
->second
.local_log_id
,
113 NotifyUploadDone(response_code
, report_id
, it
->second
);
114 upload_done_data_
.erase(it
);
120 void WebRtcLogUploader::OnURLFetchUploadProgress(
121 const net::URLFetcher
* source
, int64 current
, int64 total
) {
124 bool WebRtcLogUploader::ApplyForStartLogging() {
125 DCHECK(create_thread_checker_
.CalledOnValidThread());
126 if (log_count_
< kLogCountLimit
&& !shutting_down_
) {
133 void WebRtcLogUploader::LoggingStoppedDontUpload() {
134 content::BrowserThread::PostTask(content::BrowserThread::UI
, FROM_HERE
,
135 base::Bind(&WebRtcLogUploader::DecreaseLogCount
, base::Unretained(this)));
138 void WebRtcLogUploader::LoggingStoppedDoUpload(
139 scoped_ptr
<WebRtcLogBuffer
> log_buffer
,
140 scoped_ptr
<MetaDataMap
> meta_data
,
141 const WebRtcLogUploadDoneData
& upload_done_data
) {
142 DCHECK(file_thread_checker_
.CalledOnValidThread());
143 DCHECK(log_buffer
.get());
144 DCHECK(meta_data
.get());
145 DCHECK(!upload_done_data
.log_path
.empty());
147 std::string compressed_log
;
148 CompressLog(&compressed_log
, log_buffer
.get());
150 std::string local_log_id
;
152 if (base::PathExists(upload_done_data
.log_path
)) {
153 WebRtcLogUtil::DeleteOldWebRtcLogFiles(upload_done_data
.log_path
);
155 local_log_id
= base::DoubleToString(base::Time::Now().ToDoubleT());
156 base::FilePath log_file_path
=
157 upload_done_data
.log_path
.AppendASCII(local_log_id
)
158 .AddExtension(FILE_PATH_LITERAL(".gz"));
159 WriteCompressedLogToFile(compressed_log
, log_file_path
);
161 base::FilePath log_list_path
=
162 WebRtcLogList::GetWebRtcLogListFileForDirectory(
163 upload_done_data
.log_path
);
164 AddLocallyStoredLogInfoToUploadListFile(log_list_path
, local_log_id
);
167 WebRtcLogUploadDoneData upload_done_data_with_log_id
= upload_done_data
;
168 upload_done_data_with_log_id
.local_log_id
= local_log_id
;
169 UploadCompressedLog(compressed_log
, meta_data
.Pass(),
170 upload_done_data_with_log_id
);
173 void WebRtcLogUploader::UploadCompressedLog(
174 const std::string
& compressed_log
,
175 scoped_ptr
<MetaDataMap
> meta_data
,
176 const WebRtcLogUploadDoneData
& upload_done_data
) {
177 DCHECK(file_thread_checker_
.CalledOnValidThread());
178 DCHECK(!compressed_log
.empty());
179 DCHECK(meta_data
.get());
181 scoped_ptr
<std::string
> post_data(new std::string());
182 SetupMultipart(post_data
.get(),
184 upload_done_data
.incoming_rtp_dump
,
185 upload_done_data
.outgoing_rtp_dump
,
188 // If a test has set the test string pointer, write to it and skip uploading.
189 // Still fire the upload callback so that we can run an extension API test
190 // using the test framework for that without hanging.
191 // TODO(grunell): Remove this when the api test for this feature is fully
192 // implemented according to the test plan. http://crbug.com/257329.
194 *post_data_
= *post_data
;
195 NotifyUploadDone(kHttpResponseOk
, "", upload_done_data
);
199 content::BrowserThread::PostTask(
200 content::BrowserThread::UI
,
202 base::Bind(&WebRtcLogUploader::CreateAndStartURLFetcher
,
203 base::Unretained(this),
205 Passed(&post_data
)));
207 content::BrowserThread::PostTask(content::BrowserThread::UI
, FROM_HERE
,
208 base::Bind(&WebRtcLogUploader::DecreaseLogCount
, base::Unretained(this)));
211 void WebRtcLogUploader::UploadStoredLog(
212 const WebRtcLogUploadDoneData
& upload_data
) {
213 DCHECK(file_thread_checker_
.CalledOnValidThread());
214 DCHECK(!upload_data
.local_log_id
.empty());
215 DCHECK(!upload_data
.log_path
.empty());
217 base::FilePath native_log_path
=
218 upload_data
.log_path
.AppendASCII(upload_data
.local_log_id
)
219 .AddExtension(FILE_PATH_LITERAL(".gz"));
221 std::string compressed_log
;
222 if (!base::ReadFileToString(native_log_path
, &compressed_log
)) {
223 DPLOG(WARNING
) << "Could not read WebRTC log file.";
224 content::BrowserThread::PostTask(
225 content::BrowserThread::UI
, FROM_HERE
,
226 base::Bind(upload_data
.callback
, false, "", "Log doesn't exist."));
230 WebRtcLogUploadDoneData upload_data_with_rtp
= upload_data
;
232 // Optimistically set the rtp paths to what they should be if they exist.
233 upload_data_with_rtp
.incoming_rtp_dump
=
234 upload_data
.log_path
.AppendASCII(upload_data
.local_log_id
)
235 .AddExtension(FILE_PATH_LITERAL(".rtp_in"));
237 upload_data_with_rtp
.outgoing_rtp_dump
=
238 upload_data
.log_path
.AppendASCII(upload_data
.local_log_id
)
239 .AddExtension(FILE_PATH_LITERAL(".rtp_out"));
241 scoped_ptr
<MetaDataMap
> meta_data(new MetaDataMap());
243 std::string meta_data_contents
;
244 base::FilePath meta_path
=
245 upload_data
.log_path
.AppendASCII(upload_data
.local_log_id
)
246 .AddExtension(FILE_PATH_LITERAL(".meta"));
247 if (base::ReadFileToString(meta_path
, &meta_data_contents
) &&
248 !meta_data_contents
.empty()) {
249 base::Pickle
pickle(&meta_data_contents
[0], meta_data_contents
.size());
250 base::PickleIterator
it(pickle
);
251 std::string key
, value
;
252 while (it
.ReadString(&key
) && it
.ReadString(&value
))
253 (*meta_data
.get())[key
] = value
;
257 UploadCompressedLog(compressed_log
, meta_data
.Pass(), upload_data_with_rtp
);
260 void WebRtcLogUploader::LoggingStoppedDoStore(
261 const WebRtcLogPaths
& log_paths
,
262 const std::string
& log_id
,
263 scoped_ptr
<WebRtcLogBuffer
> log_buffer
,
264 scoped_ptr
<MetaDataMap
> meta_data
,
265 const WebRtcLoggingHandlerHost::GenericDoneCallback
& done_callback
) {
266 DCHECK(file_thread_checker_
.CalledOnValidThread());
267 DCHECK(!log_id
.empty());
268 DCHECK(log_buffer
.get());
269 DCHECK(!log_paths
.log_path
.empty());
271 WebRtcLogUtil::DeleteOldWebRtcLogFiles(log_paths
.log_path
);
273 base::FilePath log_list_path
=
274 WebRtcLogList::GetWebRtcLogListFileForDirectory(log_paths
.log_path
);
276 // Store the native log with a ".gz" extension.
277 std::string compressed_log
;
278 CompressLog(&compressed_log
, log_buffer
.get());
279 base::FilePath native_log_path
= log_paths
.log_path
.AppendASCII(log_id
)
280 .AddExtension(FILE_PATH_LITERAL(".gz"));
281 WriteCompressedLogToFile(compressed_log
, native_log_path
);
282 AddLocallyStoredLogInfoToUploadListFile(log_list_path
, log_id
);
284 // Move the rtp dump files to the log directory with a name of
285 // <log id>.rtp_[in|out].
286 if (!log_paths
.incoming_rtp_dump
.empty()) {
287 base::FilePath rtp_path
= log_paths
.log_path
.AppendASCII(log_id
)
288 .AddExtension(FILE_PATH_LITERAL(".rtp_in"));
289 base::Move(log_paths
.incoming_rtp_dump
, rtp_path
);
292 if (!log_paths
.outgoing_rtp_dump
.empty()) {
293 base::FilePath rtp_path
= log_paths
.log_path
.AppendASCII(log_id
)
294 .AddExtension(FILE_PATH_LITERAL(".rtp_out"));
295 base::Move(log_paths
.outgoing_rtp_dump
, rtp_path
);
298 if (meta_data
.get() && !meta_data
->empty()) {
300 for (const auto& it
: *meta_data
.get()) {
301 pickle
.WriteString(it
.first
);
302 pickle
.WriteString(it
.second
);
304 base::FilePath meta_path
= log_paths
.log_path
.AppendASCII(log_id
)
305 .AddExtension(FILE_PATH_LITERAL(".meta"));
306 base::WriteFile(meta_path
, static_cast<const char*>(pickle
.data()),
310 content::BrowserThread::PostTask(
311 content::BrowserThread::UI
, FROM_HERE
,
312 base::Bind(done_callback
, true, ""));
315 void WebRtcLogUploader::StartShutdown() {
316 DCHECK(create_thread_checker_
.CalledOnValidThread());
317 DCHECK(!shutting_down_
);
319 // Delete all URLFetchers first and clear the upload done map.
320 for (UploadDoneDataMap::iterator it
= upload_done_data_
.begin();
321 it
!= upload_done_data_
.end();
325 upload_done_data_
.clear();
326 shutting_down_
= true;
329 void WebRtcLogUploader::SetupMultipart(
330 std::string
* post_data
,
331 const std::string
& compressed_log
,
332 const base::FilePath
& incoming_rtp_dump
,
333 const base::FilePath
& outgoing_rtp_dump
,
334 const std::map
<std::string
, std::string
>& meta_data
) {
336 const char product
[] = "Chrome";
337 #elif defined(OS_MACOSX)
338 const char product
[] = "Chrome_Mac";
339 #elif defined(OS_LINUX)
340 #if !defined(ADDRESS_SANITIZER)
341 const char product
[] = "Chrome_Linux";
343 const char product
[] = "Chrome_Linux_ASan";
345 #elif defined(OS_ANDROID)
346 const char product
[] = "Chrome_Android";
347 #elif defined(OS_CHROMEOS)
348 const char product
[] = "Chrome_ChromeOS";
350 #error Platform not supported.
352 net::AddMultipartValueForUpload("prod", product
, kMultipartBoundary
,
354 net::AddMultipartValueForUpload("ver",
355 version_info::GetVersionNumber() + "-webrtc",
356 kMultipartBoundary
, "", post_data
);
357 net::AddMultipartValueForUpload("guid", "0", kMultipartBoundary
,
359 net::AddMultipartValueForUpload("type", "webrtc_log", kMultipartBoundary
,
362 // Add custom meta data.
363 std::map
<std::string
, std::string
>::const_iterator it
= meta_data
.begin();
364 for (; it
!= meta_data
.end(); ++it
) {
365 net::AddMultipartValueForUpload(it
->first
, it
->second
, kMultipartBoundary
,
369 AddLogData(post_data
, compressed_log
);
371 // Add the rtp dumps if they exist.
372 base::FilePath rtp_dumps
[2] = {incoming_rtp_dump
, outgoing_rtp_dump
};
373 static const char* kRtpDumpNames
[2] = {"rtpdump_recv", "rtpdump_send"};
375 for (size_t i
= 0; i
< 2; ++i
) {
376 if (!rtp_dumps
[i
].empty() && base::PathExists(rtp_dumps
[i
])) {
377 std::string dump_data
;
378 if (base::ReadFileToString(rtp_dumps
[i
], &dump_data
))
379 AddRtpDumpData(post_data
, kRtpDumpNames
[i
], dump_data
);
383 net::AddMultipartFinalDelimiterForUpload(kMultipartBoundary
, post_data
);
386 void WebRtcLogUploader::CompressLog(std::string
* compressed_log
,
387 WebRtcLogBuffer
* buffer
) {
388 z_stream stream
= {0};
389 int result
= deflateInit2(&stream
, Z_DEFAULT_COMPRESSION
, Z_DEFLATED
,
390 // windowBits = 15 is default, 16 is added to
391 // produce a gzip header + trailer.
393 8, // memLevel = 8 is default.
395 DCHECK_EQ(Z_OK
, result
);
397 uint8 intermediate_buffer
[kIntermediateCompressionBufferBytes
] = {0};
398 ResizeForNextOutput(compressed_log
, &stream
);
401 PartialCircularBuffer
read_buffer(buffer
->Read());
403 if (stream
.avail_in
== 0) {
404 read
= read_buffer
.Read(&intermediate_buffer
[0],
405 sizeof(intermediate_buffer
));
406 stream
.next_in
= &intermediate_buffer
[0];
407 stream
.avail_in
= read
;
408 if (read
!= kIntermediateCompressionBufferBytes
)
411 result
= deflate(&stream
, Z_SYNC_FLUSH
);
412 DCHECK_EQ(Z_OK
, result
);
413 if (stream
.avail_out
== 0)
414 ResizeForNextOutput(compressed_log
, &stream
);
417 // Ensure we have enough room in the output buffer. Easier to always just do a
418 // resize than looping around and resize if needed.
419 if (stream
.avail_out
< kIntermediateCompressionBufferBytes
)
420 ResizeForNextOutput(compressed_log
, &stream
);
422 result
= deflate(&stream
, Z_FINISH
);
423 DCHECK_EQ(Z_STREAM_END
, result
);
424 result
= deflateEnd(&stream
);
425 DCHECK_EQ(Z_OK
, result
);
427 compressed_log
->resize(compressed_log
->size() - stream
.avail_out
);
430 void WebRtcLogUploader::ResizeForNextOutput(std::string
* compressed_log
,
432 size_t old_size
= compressed_log
->size() - stream
->avail_out
;
433 compressed_log
->resize(old_size
+ kIntermediateCompressionBufferBytes
);
434 stream
->next_out
= reinterpret_cast<unsigned char*>(
435 &(*compressed_log
)[old_size
]);
436 stream
->avail_out
= kIntermediateCompressionBufferBytes
;
439 void WebRtcLogUploader::CreateAndStartURLFetcher(
440 const WebRtcLogUploadDoneData
& upload_done_data
,
441 scoped_ptr
<std::string
> post_data
) {
442 DCHECK(create_thread_checker_
.CalledOnValidThread());
447 std::string content_type
= kUploadContentType
;
448 content_type
.append("; boundary=");
449 content_type
.append(kMultipartBoundary
);
451 net::URLFetcher
* url_fetcher
=
452 net::URLFetcher::Create(GURL(chrome::kUploadURL
), net::URLFetcher::POST
,
454 url_fetcher
->SetRequestContext(g_browser_process
->system_request_context());
455 url_fetcher
->SetUploadData(content_type
, *post_data
);
456 url_fetcher
->Start();
457 upload_done_data_
[url_fetcher
] = upload_done_data
;
460 void WebRtcLogUploader::DecreaseLogCount() {
461 DCHECK(create_thread_checker_
.CalledOnValidThread());
465 void WebRtcLogUploader::WriteCompressedLogToFile(
466 const std::string
& compressed_log
,
467 const base::FilePath
& log_file_path
) {
468 DCHECK(file_thread_checker_
.CalledOnValidThread());
469 DCHECK(!compressed_log
.empty());
470 base::WriteFile(log_file_path
, &compressed_log
[0], compressed_log
.size());
473 void WebRtcLogUploader::AddLocallyStoredLogInfoToUploadListFile(
474 const base::FilePath
& upload_list_path
,
475 const std::string
& local_log_id
) {
476 DCHECK(file_thread_checker_
.CalledOnValidThread());
477 DCHECK(!upload_list_path
.empty());
478 DCHECK(!local_log_id
.empty());
480 std::string contents
;
482 if (base::PathExists(upload_list_path
)) {
483 if (!base::ReadFileToString(upload_list_path
, &contents
)) {
484 DPLOG(WARNING
) << "Could not read WebRTC log list file.";
488 // Limit the number of log entries to |kLogListLimitLines| - 1, to make room
489 // for the new entry. Each line including the last ends with a '\n', so hit
490 // n will be before line n-1 (from the back).
492 int i
= contents
.size() - 1;
493 for (; i
>= 0 && lf_count
< kLogListLimitLines
; --i
) {
494 if (contents
[i
] == '\n')
497 if (lf_count
>= kLogListLimitLines
) {
498 // + 1 to compensate for the for loop decrease before the conditional
499 // check and + 1 to get the length.
500 contents
.erase(0, i
+ 2);
504 // Write the log ID to the log list file. Leave the upload time and report ID
506 contents
+= ",," + local_log_id
+ '\n';
509 base::WriteFile(upload_list_path
, &contents
[0], contents
.size());
510 if (written
!= static_cast<int>(contents
.size())) {
511 DPLOG(WARNING
) << "Could not write all data to WebRTC log list file: "
516 void WebRtcLogUploader::AddUploadedLogInfoToUploadListFile(
517 const base::FilePath
& upload_list_path
,
518 const std::string
& local_log_id
,
519 const std::string
& report_id
) {
520 DCHECK(file_thread_checker_
.CalledOnValidThread());
521 DCHECK(!upload_list_path
.empty());
522 DCHECK(!local_log_id
.empty());
523 DCHECK(!report_id
.empty());
525 std::string contents
;
527 if (base::PathExists(upload_list_path
)) {
528 if (!base::ReadFileToString(upload_list_path
, &contents
)) {
529 DPLOG(WARNING
) << "Could not read WebRTC log list file.";
534 // Write the Unix time and report ID to the log list file. We should be able
535 // to find the local log ID, in that case insert the data into the existing
536 // line. Otherwise add it in the end.
537 base::Time time_now
= base::Time::Now();
538 std::string time_now_str
= base::DoubleToString(time_now
.ToDoubleT());
539 size_t pos
= contents
.find(",," + local_log_id
);
540 if (pos
!= std::string::npos
) {
541 contents
.insert(pos
, time_now_str
);
542 contents
.insert(pos
+ time_now_str
.length() + 1, report_id
);
544 contents
+= time_now_str
+ "," + report_id
+ ",\n";
548 base::WriteFile(upload_list_path
, &contents
[0], contents
.size());
549 if (written
!= static_cast<int>(contents
.size())) {
550 DPLOG(WARNING
) << "Could not write all data to WebRTC log list file: "
555 void WebRtcLogUploader::NotifyUploadDone(
557 const std::string
& report_id
,
558 const WebRtcLogUploadDoneData
& upload_done_data
) {
559 content::BrowserThread::PostTask(content::BrowserThread::IO
, FROM_HERE
,
560 base::Bind(&WebRtcLoggingHandlerHost::UploadLogDone
,
561 upload_done_data
.host
));
562 if (!upload_done_data
.callback
.is_null()) {
563 bool success
= response_code
== kHttpResponseOk
;
564 std::string error_message
;
566 error_message
= "Uploading failed, response code: " +
567 base::IntToString(response_code
);
569 content::BrowserThread::PostTask(
570 content::BrowserThread::UI
, FROM_HERE
,
571 base::Bind(upload_done_data
.callback
, success
, report_id
,