Send a crash report when a hung process is detected.
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / nacl-mono-buildbot.py
blob23645c571791a4c6d07443ab152d326250e92eba
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 import hashlib
7 import json
8 import os
9 import sys
11 import buildbot_common
12 import build_version
13 from build_paths import SCRIPT_DIR
15 GS_MANIFEST_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
16 SDK_MANIFEST = 'naclsdk_manifest2.json'
17 MONO_MANIFEST = 'naclmono_manifest.json'
19 def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url,
20 upload_path, args):
21 install_dir = 'naclmono'
22 buildbot_common.RemoveDir(install_dir)
24 revision_opt = ['--sdk-revision', sdk_revision] if sdk_revision else []
25 url_opt = ['--sdk-url', sdk_url] if sdk_url else []
27 buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
28 '--arch', 'x86-32', '--install-dir', install_dir] +
29 revision_opt + url_opt + args)
30 buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
31 '--arch', 'x86-64', '--install-dir', install_dir] +
32 revision_opt + url_opt + args)
33 buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
34 '--arch', 'arm', '--install-dir', install_dir] +
35 revision_opt + url_opt + args)
36 buildbot_common.Run([sys.executable, 'nacl-mono-archive.py',
37 '--upload-path', upload_path,
38 '--pepper-revision', pepper_revision,
39 '--install-dir', install_dir] + args)
41 def get_sdk_build_info():
42 '''Returns a list of dictionaries for versions of NaCl Mono to build which are
43 out of date compared to the SDKs available to naclsdk'''
45 # Get a copy of the naclsdk manifest file
46 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
47 GS_MANIFEST_PATH + SDK_MANIFEST, '.'])
48 manifest_file = open(SDK_MANIFEST, 'r')
49 sdk_manifest = json.loads(manifest_file.read())
50 manifest_file.close()
52 pepper_infos = []
53 for key, value in sdk_manifest.items():
54 if key == 'bundles':
55 stabilities = ['stable', 'beta', 'dev', 'post_stable']
56 # Pick pepper_* bundles, need pepper_19 or greater to build Mono
57 bundles = filter(lambda b: (b['stability'] in stabilities
58 and 'pepper_' in b['name'])
59 and b['version'] >= 19, value)
60 for b in bundles:
61 newdict = {}
62 newdict['pepper_revision'] = str(b['version'])
63 linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives'])
64 newdict['sdk_url'] = linux_arch[0]['url']
65 newdict['sdk_revision'] = b['revision']
66 newdict['stability'] = b['stability']
67 newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision']
68 pepper_infos.append(newdict)
70 # Get a copy of the naclmono manifest file
71 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
72 GS_MANIFEST_PATH + MONO_MANIFEST, '.'])
73 manifest_file = open(MONO_MANIFEST, 'r')
74 mono_manifest = json.loads(manifest_file.read())
75 manifest_file.close()
77 ret = []
78 mono_manifest_dirty = False
79 # Check to see if we need to rebuild mono based on sdk revision
80 for key, value in mono_manifest.items():
81 if key == 'bundles':
82 for info in pepper_infos:
83 bundle = filter(lambda b: b['name'] == info['naclmono_name'], value)
84 if len(bundle) == 0:
85 info['naclmono_rev'] = '1'
86 ret.append(info)
87 else:
88 if info['sdk_revision'] != bundle[0]['sdk_revision']:
89 # This bundle exists in the mono manifest, bump the revision
90 # for the new build we're about to make.
91 info['naclmono_rev'] = str(bundle[0]['revision'] + 1)
92 ret.append(info)
93 elif info['stability'] != bundle[0]['stability']:
94 # If all that happened was the SDK bundle was promoted in stability,
95 # change only that and re-write the manifest
96 mono_manifest_dirty = True
97 bundle[0]['stability'] = info['stability']
99 # re-write the manifest here because there are no bundles to build but
100 # the manifest has changed
101 if mono_manifest_dirty and len(ret) == 0:
102 manifest_file = open(MONO_MANIFEST, 'w')
103 manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
104 manifest_file.close()
105 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
106 MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
108 return ret
110 def update_mono_sdk_json(infos):
111 '''Update the naclmono manifest with the newly built packages'''
112 if len(infos) == 0:
113 return
115 manifest_file = open(MONO_MANIFEST, 'r')
116 mono_manifest = json.loads(manifest_file.read())
117 manifest_file.close()
119 for info in infos:
120 bundle = {}
121 bundle['name'] = info['naclmono_name']
122 bundle['description'] = 'Mono for Native Client'
123 bundle['stability'] = info['stability']
124 bundle['recommended'] = 'no'
125 bundle['version'] = 'experimental'
126 archive = {}
127 sha1_hash = hashlib.sha1()
128 f = open(info['naclmono_name'] + '.bz2', 'rb')
129 sha1_hash.update(f.read())
130 archive['size'] = f.tell()
131 f.close()
132 archive['checksum'] = { 'sha1': sha1_hash.hexdigest() }
133 archive['host_os'] = 'all'
134 archive['url'] = ('https://storage.googleapis.com/'
135 'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2'
136 % (info['naclmono_name'], info['naclmono_rev'],
137 info['naclmono_name']))
138 bundle['archives'] = [archive]
139 bundle['revision'] = int(info['naclmono_rev'])
140 bundle['sdk_revision'] = int(info['sdk_revision'])
142 # Insert this new bundle into the manifest,
143 # probably overwriting an existing bundle.
144 for key, value in mono_manifest.items():
145 if key == 'bundles':
146 existing = filter(lambda b: b['name'] == info['naclmono_name'], value)
147 if len(existing) > 0:
148 loc = value.index(existing[0])
149 value[loc] = bundle
150 else:
151 value.append(bundle)
153 # Write out the file locally, then upload to its known location.
154 manifest_file = open(MONO_MANIFEST, 'w')
155 manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
156 manifest_file.close()
157 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
158 MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
161 def main(args):
162 args = args[1:]
164 # Delete global configs that would override the mono builders' configuration.
165 if 'AWS_CREDENTIAL_FILE' in os.environ:
166 del os.environ['AWS_CREDENTIAL_FILE']
167 if 'BOTO_CONFIG' in os.environ:
168 del os.environ['BOTO_CONFIG']
170 buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
171 buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
173 os.chdir(SCRIPT_DIR)
175 if buildername == 'linux-sdk-mono32':
176 assert buildbot_revision
177 sdk_revision = buildbot_revision.split(':')[0]
178 pepper_revision = build_version.ChromeMajorVersion()
179 build_and_upload_mono(sdk_revision, pepper_revision, None,
180 'trunk.' + sdk_revision, args)
181 elif buildername == 'linux-sdk-mono64':
182 infos = get_sdk_build_info()
183 for info in infos:
184 # This will put the file in naclmono_19/1/naclmono_19.bz2 for example.
185 upload_path = info['naclmono_name'] + '/' + info['naclmono_rev']
186 build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'],
187 upload_path, args)
188 update_mono_sdk_json(infos)
192 if __name__ == '__main__':
193 sys.exit(main(sys.argv))