build/android/pylib/results/flakiness_dashboard/json_results_generator.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 #
   6 # Most of this file was ported over from Blink's
   7 # Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
   8 # Tools/Scripts/webkitpy/common/net/file_uploader.py
   9 #
  10
  11 import json
  12 import logging
  13 import mimetypes
  14 import os
  15 import time
  16 import urllib2
  17
  18 _log = logging.getLogger(__name__)
  19
  20 _JSON_PREFIX = 'ADD_RESULTS('
  21 _JSON_SUFFIX = ');'
  22
  23
  24 def HasJSONWrapper(string):
  25   return string.startswith(_JSON_PREFIX) and string.endswith(_JSON_SUFFIX)
  26
  27
  28 def StripJSONWrapper(json_content):
  29   # FIXME: Kill this code once the server returns json instead of jsonp.
  30   if HasJSONWrapper(json_content):
  31     return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)]
  32   return json_content
  33
  34
  35 def WriteJSON(json_object, file_path, callback=None):
  36   # Specify separators in order to get compact encoding.
  37   json_string = json.dumps(json_object, separators=(',', ':'))
  38   if callback:
  39     json_string = callback + '(' + json_string + ');'
  40   with open(file_path, 'w') as fp:
  41     fp.write(json_string)
  42
  43
  44 def ConvertTrieToFlatPaths(trie, prefix=None):
  45   """Flattens the trie of paths, prepending a prefix to each."""
  46   result = {}
  47   for name, data in trie.iteritems():
  48     if prefix:
  49       name = prefix + '/' + name
  50
  51     if len(data) and not 'results' in data:
  52       result.update(ConvertTrieToFlatPaths(data, name))
  53     else:
  54       result[name] = data
  55
  56   return result
  57
  58
  59 def AddPathToTrie(path, value, trie):
  60   """Inserts a single path and value into a directory trie structure."""
  61   if not '/' in path:
  62     trie[path] = value
  63     return
  64
  65   directory, _slash, rest = path.partition('/')
  66   if not directory in trie:
  67     trie[directory] = {}
  68   AddPathToTrie(rest, value, trie[directory])
  69
  70
  71 def TestTimingsTrie(individual_test_timings):
  72   """Breaks a test name into dicts by directory
  73
  74   foo/bar/baz.html: 1ms
  75   foo/bar/baz1.html: 3ms
  76
  77   becomes
  78   foo: {
  79       bar: {
  80           baz.html: 1,
  81           baz1.html: 3
  82       }
  83   }
  84   """
  85   trie = {}
  86   for test_result in individual_test_timings:
  87     test = test_result.test_name
  88
  89     AddPathToTrie(test, int(1000 * test_result.test_run_time), trie)
  90
  91   return trie
  92
  93
  94 class TestResult(object):
  95   """A simple class that represents a single test result."""
  96
  97   # Test modifier constants.
  98   (NONE, FAILS, FLAKY, DISABLED) = range(4)
  99
 100   def __init__(self, test, failed=False, elapsed_time=0):
 101     self.test_name = test
 102     self.failed = failed
 103     self.test_run_time = elapsed_time
 104
 105     test_name = test
 106     try:
 107       test_name = test.split('.')[1]
 108     except IndexError:
 109       _log.warn('Invalid test name: %s.', test)
 110
 111     if test_name.startswith('FAILS_'):
 112       self.modifier = self.FAILS
 113     elif test_name.startswith('FLAKY_'):
 114       self.modifier = self.FLAKY
 115     elif test_name.startswith('DISABLED_'):
 116       self.modifier = self.DISABLED
 117     else:
 118       self.modifier = self.NONE
 119
 120   def Fixable(self):
 121     return self.failed or self.modifier == self.DISABLED
 122
 123
 124 class JSONResultsGeneratorBase(object):
 125   """A JSON results generator for generic tests."""
 126
 127   MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750
 128   # Min time (seconds) that will be added to the JSON.
 129   MIN_TIME = 1
 130
 131   # Note that in non-chromium tests those chars are used to indicate
 132   # test modifiers (FAILS, FLAKY, etc) but not actual test results.
 133   PASS_RESULT = 'P'
 134   SKIP_RESULT = 'X'
 135   FAIL_RESULT = 'F'
 136   FLAKY_RESULT = 'L'
 137   NO_DATA_RESULT = 'N'
 138
 139   MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT,
 140                       TestResult.DISABLED: SKIP_RESULT,
 141                       TestResult.FAILS: FAIL_RESULT,
 142                       TestResult.FLAKY: FLAKY_RESULT}
 143
 144   VERSION = 4
 145   VERSION_KEY = 'version'
 146   RESULTS = 'results'
 147   TIMES = 'times'
 148   BUILD_NUMBERS = 'buildNumbers'
 149   TIME = 'secondsSinceEpoch'
 150   TESTS = 'tests'
 151
 152   FIXABLE_COUNT = 'fixableCount'
 153   FIXABLE = 'fixableCounts'
 154   ALL_FIXABLE_COUNT = 'allFixableCount'
 155
 156   RESULTS_FILENAME = 'results.json'
 157   TIMES_MS_FILENAME = 'times_ms.json'
 158   INCREMENTAL_RESULTS_FILENAME = 'incremental_results.json'
 159
 160   # line too long pylint: disable=line-too-long
 161   URL_FOR_TEST_LIST_JSON = (
 162       'http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s&master=%s')
 163   # pylint: enable=line-too-long
 164
 165   def __init__(self, builder_name, build_name, build_number,
 166                results_file_base_path, builder_base_url,
 167                test_results_map, svn_repositories=None,
 168                test_results_server=None,
 169                test_type='',
 170                master_name=''):
 171     """Modifies the results.json file. Grabs it off the archive directory
 172     if it is not found locally.
 173
 174     Args
 175       builder_name: the builder name (e.g. Webkit).
 176       build_name: the build name (e.g. webkit-rel).
 177       build_number: the build number.
 178       results_file_base_path: Absolute path to the directory containing the
 179           results json file.
 180       builder_base_url: the URL where we have the archived test results.
 181           If this is None no archived results will be retrieved.
 182       test_results_map: A dictionary that maps test_name to TestResult.
 183       svn_repositories: A (json_field_name, svn_path) pair for SVN
 184           repositories that tests rely on.  The SVN revision will be
 185           included in the JSON with the given json_field_name.
 186       test_results_server: server that hosts test results json.
 187       test_type: test type string (e.g. 'layout-tests').
 188       master_name: the name of the buildbot master.
 189     """
 190     self._builder_name = builder_name
 191     self._build_name = build_name
 192     self._build_number = build_number
 193     self._builder_base_url = builder_base_url
 194     self._results_directory = results_file_base_path
 195
 196     self._test_results_map = test_results_map
 197     self._test_results = test_results_map.values()
 198
 199     self._svn_repositories = svn_repositories
 200     if not self._svn_repositories:
 201       self._svn_repositories = {}
 202
 203     self._test_results_server = test_results_server
 204     self._test_type = test_type
 205     self._master_name = master_name
 206
 207     self._archived_results = None
 208
 209   def GenerateJSONOutput(self):
 210     json_object = self.GetJSON()
 211     if json_object:
 212       file_path = (
 213           os.path.join(
 214               self._results_directory,
 215               self.INCREMENTAL_RESULTS_FILENAME))
 216       WriteJSON(json_object, file_path)
 217
 218   def GenerateTimesMSFile(self):
 219     times = TestTimingsTrie(self._test_results_map.values())
 220     file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME)
 221     WriteJSON(times, file_path)
 222
 223   def GetJSON(self):
 224     """Gets the results for the results.json file."""
 225     results_json = {}
 226
 227     if not results_json:
 228       results_json, error = self._GetArchivedJSONResults()
 229       if error:
 230         # If there was an error don't write a results.json
 231         # file at all as it would lose all the information on the
 232         # bot.
 233         _log.error('Archive directory is inaccessible. Not '
 234                    'modifying or clobbering the results.json '
 235                    'file: ' + str(error))
 236         return None
 237
 238     builder_name = self._builder_name
 239     if results_json and builder_name not in results_json:
 240       _log.debug('Builder name (%s) is not in the results.json file.'
 241                  % builder_name)
 242
 243     self._ConvertJSONToCurrentVersion(results_json)
 244
 245     if builder_name not in results_json:
 246       results_json[builder_name] = (
 247           self._CreateResultsForBuilderJSON())
 248
 249     results_for_builder = results_json[builder_name]
 250
 251     if builder_name:
 252       self._InsertGenericMetaData(results_for_builder)
 253
 254     self._InsertFailureSummaries(results_for_builder)
 255
 256     # Update the all failing tests with result type and time.
 257     tests = results_for_builder[self.TESTS]
 258     all_failing_tests = self._GetFailedTestNames()
 259     all_failing_tests.update(ConvertTrieToFlatPaths(tests))
 260
 261     for test in all_failing_tests:
 262       self._InsertTestTimeAndResult(test, tests)
 263
 264     return results_json
 265
 266   def SetArchivedResults(self, archived_results):
 267     self._archived_results = archived_results
 268
 269   def UploadJSONFiles(self, json_files):
 270     """Uploads the given json_files to the test_results_server (if the
 271     test_results_server is given)."""
 272     if not self._test_results_server:
 273       return
 274
 275     if not self._master_name:
 276       _log.error(
 277           '--test-results-server was set, but --master-name was not.  Not '
 278           'uploading JSON files.')
 279       return
 280
 281     _log.info('Uploading JSON files for builder: %s', self._builder_name)
 282     attrs = [('builder', self._builder_name),
 283              ('testtype', self._test_type),
 284              ('master', self._master_name)]
 285
 286     files = [(json_file, os.path.join(self._results_directory, json_file))
 287              for json_file in json_files]
 288
 289     url = 'http://%s/testfile/upload' % self._test_results_server
 290     # Set uploading timeout in case appengine server is having problems.
 291     # 120 seconds are more than enough to upload test results.
 292     uploader = _FileUploader(url, 120)
 293     try:
 294       response = uploader.UploadAsMultipartFormData(files, attrs)
 295       if response:
 296         if response.code == 200:
 297           _log.info('JSON uploaded.')
 298         else:
 299           _log.debug(
 300               "JSON upload failed, %d: '%s'" %
 301               (response.code, response.read()))
 302       else:
 303         _log.error('JSON upload failed; no response returned')
 304     except Exception, err:
 305       _log.error('Upload failed: %s' % err)
 306       return
 307
 308   def _GetTestTiming(self, test_name):
 309     """Returns test timing data (elapsed time) in second
 310     for the given test_name."""
 311     if test_name in self._test_results_map:
 312       # Floor for now to get time in seconds.
 313       return int(self._test_results_map[test_name].test_run_time)
 314     return 0
 315
 316   def _GetFailedTestNames(self):
 317     """Returns a set of failed test names."""
 318     return set([r.test_name for r in self._test_results if r.failed])
 319
 320   def _GetModifierChar(self, test_name):
 321     """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT,
 322     PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier
 323     for the given test_name.
 324     """
 325     if test_name not in self._test_results_map:
 326       return self.__class__.NO_DATA_RESULT
 327
 328     test_result = self._test_results_map[test_name]
 329     if test_result.modifier in self.MODIFIER_TO_CHAR.keys():
 330       return self.MODIFIER_TO_CHAR[test_result.modifier]
 331
 332     return self.__class__.PASS_RESULT
 333
 334   def _get_result_char(self, test_name):
 335     """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT,
 336     PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result
 337     for the given test_name.
 338     """
 339     if test_name not in self._test_results_map:
 340       return self.__class__.NO_DATA_RESULT
 341
 342     test_result = self._test_results_map[test_name]
 343     if test_result.modifier == TestResult.DISABLED:
 344       return self.__class__.SKIP_RESULT
 345
 346     if test_result.failed:
 347       return self.__class__.FAIL_RESULT
 348
 349     return self.__class__.PASS_RESULT
 350
 351   def _GetSVNRevision(self, in_directory):
 352     """Returns the svn revision for the given directory.
 353
 354     Args:
 355       in_directory: The directory where svn is to be run.
 356     """
 357     # This is overridden in flakiness_dashboard_results_uploader.py.
 358     raise NotImplementedError()
 359
 360   def _GetArchivedJSONResults(self):
 361     """Download JSON file that only contains test
 362     name list from test-results server. This is for generating incremental
 363     JSON so the file generated has info for tests that failed before but
 364     pass or are skipped from current run.
 365
 366     Returns (archived_results, error) tuple where error is None if results
 367     were successfully read.
 368     """
 369     results_json = {}
 370     old_results = None
 371     error = None
 372
 373     if not self._test_results_server:
 374       return {}, None
 375
 376     results_file_url = (self.URL_FOR_TEST_LIST_JSON %
 377                         (urllib2.quote(self._test_results_server),
 378                          urllib2.quote(self._builder_name),
 379                          self.RESULTS_FILENAME,
 380                          urllib2.quote(self._test_type),
 381                          urllib2.quote(self._master_name)))
 382
 383     try:
 384       # FIXME: We should talk to the network via a Host object.
 385       results_file = urllib2.urlopen(results_file_url)
 386       old_results = results_file.read()
 387     except urllib2.HTTPError, http_error:
 388       # A non-4xx status code means the bot is hosed for some reason
 389       # and we can't grab the results.json file off of it.
 390       if http_error.code < 400 and http_error.code >= 500:
 391         error = http_error
 392     except urllib2.URLError, url_error:
 393       error = url_error
 394
 395     if old_results:
 396       # Strip the prefix and suffix so we can get the actual JSON object.
 397       old_results = StripJSONWrapper(old_results)
 398
 399       try:
 400         results_json = json.loads(old_results)
 401       except Exception:
 402         _log.debug('results.json was not valid JSON. Clobbering.')
 403         # The JSON file is not valid JSON. Just clobber the results.
 404         results_json = {}
 405     else:
 406       _log.debug('Old JSON results do not exist. Starting fresh.')
 407       results_json = {}
 408
 409     return results_json, error
 410
 411   def _InsertFailureSummaries(self, results_for_builder):
 412     """Inserts aggregate pass/failure statistics into the JSON.
 413     This method reads self._test_results and generates
 414     FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries.
 415
 416     Args:
 417       results_for_builder: Dictionary containing the test results for a
 418           single builder.
 419     """
 420     # Insert the number of tests that failed or skipped.
 421     fixable_count = len([r for r in self._test_results if r.Fixable()])
 422     self._InsertItemIntoRawList(results_for_builder,
 423                                 fixable_count, self.FIXABLE_COUNT)
 424
 425     # Create a test modifiers (FAILS, FLAKY etc) summary dictionary.
 426     entry = {}
 427     for test_name in self._test_results_map.iterkeys():
 428       result_char = self._GetModifierChar(test_name)
 429       entry[result_char] = entry.get(result_char, 0) + 1
 430
 431     # Insert the pass/skip/failure summary dictionary.
 432     self._InsertItemIntoRawList(results_for_builder, entry,
 433                                 self.FIXABLE)
 434
 435     # Insert the number of all the tests that are supposed to pass.
 436     all_test_count = len(self._test_results)
 437     self._InsertItemIntoRawList(results_for_builder,
 438                                 all_test_count, self.ALL_FIXABLE_COUNT)
 439
 440   def _InsertItemIntoRawList(self, results_for_builder, item, key):
 441     """Inserts the item into the list with the given key in the results for
 442     this builder. Creates the list if no such list exists.
 443
 444     Args:
 445       results_for_builder: Dictionary containing the test results for a
 446           single builder.
 447       item: Number or string to insert into the list.
 448       key: Key in results_for_builder for the list to insert into.
 449     """
 450     if key in results_for_builder:
 451       raw_list = results_for_builder[key]
 452     else:
 453       raw_list = []
 454
 455     raw_list.insert(0, item)
 456     raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG]
 457     results_for_builder[key] = raw_list
 458
 459   def _InsertItemRunLengthEncoded(self, item, encoded_results):
 460     """Inserts the item into the run-length encoded results.
 461
 462     Args:
 463       item: String or number to insert.
 464       encoded_results: run-length encoded results. An array of arrays, e.g.
 465           [[3,'A'],[1,'Q']] encodes AAAQ.
 466     """
 467     if len(encoded_results) and item == encoded_results[0][1]:
 468       num_results = encoded_results[0][0]
 469       if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
 470         encoded_results[0][0] = num_results + 1
 471     else:
 472       # Use a list instead of a class for the run-length encoding since
 473       # we want the serialized form to be concise.
 474       encoded_results.insert(0, [1, item])
 475
 476   def _InsertGenericMetaData(self, results_for_builder):
 477     """ Inserts generic metadata (such as version number, current time etc)
 478     into the JSON.
 479
 480     Args:
 481       results_for_builder: Dictionary containing the test results for
 482           a single builder.
 483     """
 484     self._InsertItemIntoRawList(results_for_builder,
 485                                 self._build_number, self.BUILD_NUMBERS)
 486
 487     # Include SVN revisions for the given repositories.
 488     for (name, path) in self._svn_repositories:
 489       # Note: for JSON file's backward-compatibility we use 'chrome' rather
 490       # than 'chromium' here.
 491       lowercase_name = name.lower()
 492       if lowercase_name == 'chromium':
 493         lowercase_name = 'chrome'
 494       self._InsertItemIntoRawList(results_for_builder,
 495                                   self._GetSVNRevision(path),
 496                                   lowercase_name + 'Revision')
 497
 498     self._InsertItemIntoRawList(results_for_builder,
 499                                 int(time.time()),
 500                                 self.TIME)
 501
 502   def _InsertTestTimeAndResult(self, test_name, tests):
 503     """ Insert a test item with its results to the given tests dictionary.
 504
 505     Args:
 506       tests: Dictionary containing test result entries.
 507     """
 508
 509     result = self._get_result_char(test_name)
 510     test_time = self._GetTestTiming(test_name)
 511
 512     this_test = tests
 513     for segment in test_name.split('/'):
 514       if segment not in this_test:
 515         this_test[segment] = {}
 516       this_test = this_test[segment]
 517
 518     if not len(this_test):
 519       self._PopulateResultsAndTimesJSON(this_test)
 520
 521     if self.RESULTS in this_test:
 522       self._InsertItemRunLengthEncoded(result, this_test[self.RESULTS])
 523     else:
 524       this_test[self.RESULTS] = [[1, result]]
 525
 526     if self.TIMES in this_test:
 527       self._InsertItemRunLengthEncoded(test_time, this_test[self.TIMES])
 528     else:
 529       this_test[self.TIMES] = [[1, test_time]]
 530
 531   def _ConvertJSONToCurrentVersion(self, results_json):
 532     """If the JSON does not match the current version, converts it to the
 533     current version and adds in the new version number.
 534     """
 535     if self.VERSION_KEY in results_json:
 536       archive_version = results_json[self.VERSION_KEY]
 537       if archive_version == self.VERSION:
 538         return
 539     else:
 540       archive_version = 3
 541
 542     # version 3->4
 543     if archive_version == 3:
 544       for results in results_json.values():
 545         self._ConvertTestsToTrie(results)
 546
 547     results_json[self.VERSION_KEY] = self.VERSION
 548
 549   def _ConvertTestsToTrie(self, results):
 550     if not self.TESTS in results:
 551       return
 552
 553     test_results = results[self.TESTS]
 554     test_results_trie = {}
 555     for test in test_results.iterkeys():
 556       single_test_result = test_results[test]
 557       AddPathToTrie(test, single_test_result, test_results_trie)
 558
 559     results[self.TESTS] = test_results_trie
 560
 561   def _PopulateResultsAndTimesJSON(self, results_and_times):
 562     results_and_times[self.RESULTS] = []
 563     results_and_times[self.TIMES] = []
 564     return results_and_times
 565
 566   def _CreateResultsForBuilderJSON(self):
 567     results_for_builder = {}
 568     results_for_builder[self.TESTS] = {}
 569     return results_for_builder
 570
 571   def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list):
 572     """Removes items from the run-length encoded list after the final
 573     item that exceeds the max number of builds to track.
 574
 575     Args:
 576       encoded_results: run-length encoded results. An array of arrays, e.g.
 577           [[3,'A'],[1,'Q']] encodes AAAQ.
 578     """
 579     num_builds = 0
 580     index = 0
 581     for result in encoded_list:
 582       num_builds = num_builds + result[0]
 583       index = index + 1
 584       if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
 585         return encoded_list[:index]
 586     return encoded_list
 587
 588   def _NormalizeResultsJSON(self, test, test_name, tests):
 589     """ Prune tests where all runs pass or tests that no longer exist and
 590     truncate all results to maxNumberOfBuilds.
 591
 592     Args:
 593       test: ResultsAndTimes object for this test.
 594       test_name: Name of the test.
 595       tests: The JSON object with all the test results for this builder.
 596     """
 597     test[self.RESULTS] = self._RemoveItemsOverMaxNumberOfBuilds(
 598         test[self.RESULTS])
 599     test[self.TIMES] = self._RemoveItemsOverMaxNumberOfBuilds(
 600         test[self.TIMES])
 601
 602     is_all_pass = self._IsResultsAllOfType(test[self.RESULTS],
 603                                            self.PASS_RESULT)
 604     is_all_no_data = self._IsResultsAllOfType(test[self.RESULTS],
 605                                               self.NO_DATA_RESULT)
 606     max_time = max([test_time[1] for test_time in test[self.TIMES]])
 607
 608     # Remove all passes/no-data from the results to reduce noise and
 609     # filesize. If a test passes every run, but takes > MIN_TIME to run,
 610     # don't throw away the data.
 611     if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME):
 612       del tests[test_name]
 613
 614   # method could be a function pylint: disable=R0201
 615   def _IsResultsAllOfType(self, results, result_type):
 616     """Returns whether all the results are of the given type
 617     (e.g. all passes)."""
 618     return len(results) == 1 and results[0][1] == result_type
 619
 620
 621 class _FileUploader(object):
 622
 623   def __init__(self, url, timeout_seconds):
 624     self._url = url
 625     self._timeout_seconds = timeout_seconds
 626
 627   def UploadAsMultipartFormData(self, files, attrs):
 628     file_objs = []
 629     for filename, path in files:
 630       with file(path, 'rb') as fp:
 631         file_objs.append(('file', filename, fp.read()))
 632
 633     # FIXME: We should use the same variable names for the formal and actual
 634     # parameters.
 635     content_type, data = _EncodeMultipartFormData(attrs, file_objs)
 636     return self._UploadData(content_type, data)
 637
 638   def _UploadData(self, content_type, data):
 639     start = time.time()
 640     end = start + self._timeout_seconds
 641     while time.time() < end:
 642       try:
 643         request = urllib2.Request(self._url, data,
 644                                   {'Content-Type': content_type})
 645         return urllib2.urlopen(request)
 646       except urllib2.HTTPError as e:
 647         _log.warn("Received HTTP status %s loading \"%s\".  "
 648                   'Retrying in 10 seconds...' % (e.code, e.filename))
 649         time.sleep(10)
 650
 651
 652 def _GetMIMEType(filename):
 653   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
 654
 655
 656 # FIXME: Rather than taking tuples, this function should take more
 657 # structured data.
 658 def _EncodeMultipartFormData(fields, files):
 659   """Encode form fields for multipart/form-data.
 660
 661   Args:
 662     fields: A sequence of (name, value) elements for regular form fields.
 663     files: A sequence of (name, filename, value) elements for data to be
 664            uploaded as files.
 665   Returns:
 666     (content_type, body) ready for httplib.HTTP instance.
 667
 668   Source:
 669     http://code.google.com/p/rietveld/source/browse/trunk/upload.py
 670   """
 671   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
 672   CRLF = '\r\n'
 673   lines = []
 674
 675   for key, value in fields:
 676     lines.append('--' + BOUNDARY)
 677     lines.append('Content-Disposition: form-data; name="%s"' % key)
 678     lines.append('')
 679     if isinstance(value, unicode):
 680       value = value.encode('utf-8')
 681     lines.append(value)
 682
 683   for key, filename, value in files:
 684     lines.append('--' + BOUNDARY)
 685     lines.append('Content-Disposition: form-data; name="%s"; '
 686                  'filename="%s"' % (key, filename))
 687     lines.append('Content-Type: %s' % _GetMIMEType(filename))
 688     lines.append('')
 689     if isinstance(value, unicode):
 690       value = value.encode('utf-8')
 691     lines.append(value)
 692
 693   lines.append('--' + BOUNDARY + '--')
 694   lines.append('')
 695   body = CRLF.join(lines)
 696   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
 697   return content_type, body