components/domain_reliability/bake_in_configs.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6
   7 """Takes the JSON files in components/domain_reliability/baked_in_configs and
   8 encodes their contents as an array of C strings that gets compiled in to Chrome
   9 and loaded at runtime."""
  10
  11
  12 import ast
  13 import json
  14 import os
  15 import sys
  16
  17
  18 # A whitelist of domains that the script will accept when baking configs in to
  19 # Chrome, to ensure incorrect ones are not added accidentally. Subdomains of
  20 # whitelist entries are also allowed (e.g. maps.google.com, ssl.gstatic.com).
  21 DOMAIN_WHITELIST = (
  22   '2mdn.net',
  23   'admob.biz',
  24   'admob.co.in',
  25   'admob.co.kr',
  26   'admob.co.nz',
  27   'admob.co.uk',
  28   'admob.co.za',
  29   'admob.com',
  30   'admob.com.br',
  31   'admob.com.es',
  32   'admob.com.fr',
  33   'admob.com.mx',
  34   'admob.com.pt',
  35   'admob.de',
  36   'admob.dk',
  37   'admob.es',
  38   'admob.fi',
  39   'admob.fr',
  40   'admob.gr',
  41   'admob.hk',
  42   'admob.ie',
  43   'admob.in',
  44   'admob.it',
  45   'admob.jp',
  46   'admob.kr',
  47   'admob.mobi',
  48   'admob.no',
  49   'admob.ph',
  50   'admob.pt',
  51   'admob.sg',
  52   'admob.tw',
  53   'admob.us',
  54   'admob.vn',
  55   'dartmotif.com',
  56   'doubleclick.com',
  57   'doubleclick.ne.jp',
  58   'doubleclick.net',
  59   'doubleclickusercontent.com',
  60   'g.co',
  61   'ggpht.com',
  62   'gmodules.com',
  63   'goo.gl',
  64   'google-analytics.com',
  65   'google-syndication.com',
  66   'google.ac',
  67   'google.ad',
  68   'google.ae',
  69   'google.af',
  70   'google.ag',
  71   'google.al',
  72   'google.am',
  73   'google.as',
  74   'google.at',
  75   'google.az',
  76   'google.ba',
  77   'google.be',
  78   'google.bf',
  79   'google.bg',
  80   'google.bi',
  81   'google.bj',
  82   'google.bs',
  83   'google.bt',
  84   'google.by',
  85   'google.ca',
  86   'google.cat',
  87   'google.cc',
  88   'google.cd',
  89   'google.cf',
  90   'google.cg',
  91   'google.ch',
  92   'google.ci',
  93   'google.cl',
  94   'google.cm',
  95   'google.cn',
  96   'google.co.ao',
  97   'google.co.bw',
  98   'google.co.ck',
  99   'google.co.cr',
 100   'google.co.hu',
 101   'google.co.id',
 102   'google.co.il',
 103   'google.co.im',
 104   'google.co.in',
 105   'google.co.je',
 106   'google.co.jp',
 107   'google.co.ke',
 108   'google.co.kr',
 109   'google.co.ls',
 110   'google.co.ma',
 111   'google.co.mz',
 112   'google.co.nz',
 113   'google.co.th',
 114   'google.co.tz',
 115   'google.co.ug',
 116   'google.co.uk',
 117   'google.co.uz',
 118   'google.co.ve',
 119   'google.co.vi',
 120   'google.co.za',
 121   'google.co.zm',
 122   'google.co.zw',
 123   'google.com',
 124   'google.com.af',
 125   'google.com.ag',
 126   'google.com.ai',
 127   'google.com.ar',
 128   'google.com.au',
 129   'google.com.bd',
 130   'google.com.bh',
 131   'google.com.bn',
 132   'google.com.bo',
 133   'google.com.br',
 134   'google.com.by',
 135   'google.com.bz',
 136   'google.com.cn',
 137   'google.com.co',
 138   'google.com.cu',
 139   'google.com.cy',
 140   'google.com.do',
 141   'google.com.ec',
 142   'google.com.eg',
 143   'google.com.et',
 144   'google.com.fj',
 145   'google.com.ge',
 146   'google.com.gh',
 147   'google.com.gi',
 148   'google.com.gr',
 149   'google.com.gt',
 150   'google.com.hk',
 151   'google.com.iq',
 152   'google.com.jm',
 153   'google.com.jo',
 154   'google.com.kh',
 155   'google.com.kw',
 156   'google.com.lb',
 157   'google.com.ly',
 158   'google.com.mm',
 159   'google.com.mt',
 160   'google.com.mx',
 161   'google.com.my',
 162   'google.com.na',
 163   'google.com.nf',
 164   'google.com.ng',
 165   'google.com.ni',
 166   'google.com.np',
 167   'google.com.nr',
 168   'google.com.om',
 169   'google.com.pa',
 170   'google.com.pe',
 171   'google.com.pg',
 172   'google.com.ph',
 173   'google.com.pk',
 174   'google.com.pl',
 175   'google.com.pr',
 176   'google.com.py',
 177   'google.com.qa',
 178   'google.com.ru',
 179   'google.com.sa',
 180   'google.com.sb',
 181   'google.com.sg',
 182   'google.com.sl',
 183   'google.com.sv',
 184   'google.com.tj',
 185   'google.com.tn',
 186   'google.com.tr',
 187   'google.com.tw',
 188   'google.com.ua',
 189   'google.com.uy',
 190   'google.com.vc',
 191   'google.com.ve',
 192   'google.com.vn',
 193   'google.cv',
 194   'google.cz',
 195   'google.de',
 196   'google.dj',
 197   'google.dk',
 198   'google.dm',
 199   'google.dz',
 200   'google.ee',
 201   'google.es',
 202   'google.fi',
 203   'google.fm',
 204   'google.fr',
 205   'google.ga',
 206   'google.ge',
 207   'google.gg',
 208   'google.gl',
 209   'google.gm',
 210   'google.gp',
 211   'google.gr',
 212   'google.gy',
 213   'google.hk',
 214   'google.hn',
 215   'google.hr',
 216   'google.ht',
 217   'google.hu',
 218   'google.ie',
 219   'google.im',
 220   'google.info',
 221   'google.iq',
 222   'google.ir',
 223   'google.is',
 224   'google.it',
 225   'google.it.ao',
 226   'google.je',
 227   'google.jo',
 228   'google.jobs',
 229   'google.jp',
 230   'google.kg',
 231   'google.ki',
 232   'google.kz',
 233   'google.la',
 234   'google.li',
 235   'google.lk',
 236   'google.lt',
 237   'google.lu',
 238   'google.lv',
 239   'google.md',
 240   'google.me',
 241   'google.mg',
 242   'google.mk',
 243   'google.ml',
 244   'google.mn',
 245   'google.ms',
 246   'google.mu',
 247   'google.mv',
 248   'google.mw',
 249   'google.ne',
 250   'google.ne.jp',
 251   'google.net',
 252   'google.ng',
 253   'google.nl',
 254   'google.no',
 255   'google.nr',
 256   'google.nu',
 257   'google.off.ai',
 258   'google.org',
 259   'google.pk',
 260   'google.pl',
 261   'google.pn',
 262   'google.ps',
 263   'google.pt',
 264   'google.ro',
 265   'google.rs',
 266   'google.ru',
 267   'google.rw',
 268   'google.sc',
 269   'google.se',
 270   'google.sh',
 271   'google.si',
 272   'google.sk',
 273   'google.sm',
 274   'google.sn',
 275   'google.so',
 276   'google.sr',
 277   'google.st',
 278   'google.td',
 279   'google.tg',
 280   'google.tk',
 281   'google.tl',
 282   'google.tm',
 283   'google.tn',
 284   'google.to',
 285   'google.tt',
 286   'google.us',
 287   'google.uz',
 288   'google.vg',
 289   'google.vu',
 290   'google.ws',
 291   'googleadservices.com',
 292   'googlealumni.com',
 293   'googleapis.com',
 294   'googleapps.com',
 295   'googlecbs.com',
 296   'googlecommerce.com',
 297   'googledrive.com',
 298   'googleenterprise.com',
 299   'googlegoro.com',
 300   'googlehosted.com',
 301   'googlepayments.com',
 302   'googlesource.com',
 303   'googlesyndication.com',
 304   'googletagmanager.com',
 305   'googletagservices.com',
 306   'googleusercontent.com',
 307   'googlevideo.com',
 308   'gstatic.com',
 309   'gvt1.com',
 310   'gvt2.com',
 311   'withgoogle.com',
 312   'youtu.be',
 313   'youtube-3rd-party.com',
 314   'youtube-nocookie.com',
 315   'youtube.ae',
 316   'youtube.al',
 317   'youtube.am',
 318   'youtube.at',
 319   'youtube.az',
 320   'youtube.ba',
 321   'youtube.be',
 322   'youtube.bg',
 323   'youtube.bh',
 324   'youtube.bo',
 325   'youtube.ca',
 326   'youtube.cat',
 327   'youtube.ch',
 328   'youtube.cl',
 329   'youtube.co',
 330   'youtube.co.ae',
 331   'youtube.co.at',
 332   'youtube.co.hu',
 333   'youtube.co.id',
 334   'youtube.co.il',
 335   'youtube.co.in',
 336   'youtube.co.jp',
 337   'youtube.co.ke',
 338   'youtube.co.kr',
 339   'youtube.co.ma',
 340   'youtube.co.nz',
 341   'youtube.co.th',
 342   'youtube.co.ug',
 343   'youtube.co.uk',
 344   'youtube.co.ve',
 345   'youtube.co.za',
 346   'youtube.com',
 347   'youtube.com.ar',
 348   'youtube.com.au',
 349   'youtube.com.az',
 350   'youtube.com.bh',
 351   'youtube.com.bo',
 352   'youtube.com.br',
 353   'youtube.com.by',
 354   'youtube.com.co',
 355   'youtube.com.do',
 356   'youtube.com.ee',
 357   'youtube.com.eg',
 358   'youtube.com.es',
 359   'youtube.com.gh',
 360   'youtube.com.gr',
 361   'youtube.com.gt',
 362   'youtube.com.hk',
 363   'youtube.com.hr',
 364   'youtube.com.jm',
 365   'youtube.com.jo',
 366   'youtube.com.kw',
 367   'youtube.com.lb',
 368   'youtube.com.lv',
 369   'youtube.com.mk',
 370   'youtube.com.mt',
 371   'youtube.com.mx',
 372   'youtube.com.my',
 373   'youtube.com.ng',
 374   'youtube.com.om',
 375   'youtube.com.pe',
 376   'youtube.com.ph',
 377   'youtube.com.pk',
 378   'youtube.com.pt',
 379   'youtube.com.qa',
 380   'youtube.com.ro',
 381   'youtube.com.sa',
 382   'youtube.com.sg',
 383   'youtube.com.tn',
 384   'youtube.com.tr',
 385   'youtube.com.tw',
 386   'youtube.com.ua',
 387   'youtube.com.uy',
 388   'youtube.com.ve',
 389   'youtube.cz',
 390   'youtube.de',
 391   'youtube.dk',
 392   'youtube.ee',
 393   'youtube.es',
 394   'youtube.fi',
 395   'youtube.fr',
 396   'youtube.ge',
 397   'youtube.gr',
 398   'youtube.gt',
 399   'youtube.hk',
 400   'youtube.hr',
 401   'youtube.hu',
 402   'youtube.ie',
 403   'youtube.in',
 404   'youtube.is',
 405   'youtube.it',
 406   'youtube.jo',
 407   'youtube.jp',
 408   'youtube.kr',
 409   'youtube.lk',
 410   'youtube.lt',
 411   'youtube.lv',
 412   'youtube.ma',
 413   'youtube.md',
 414   'youtube.me',
 415   'youtube.mk',
 416   'youtube.mx',
 417   'youtube.my',
 418   'youtube.ng',
 419   'youtube.nl',
 420   'youtube.no',
 421   'youtube.pe',
 422   'youtube.ph',
 423   'youtube.pk',
 424   'youtube.pl',
 425   'youtube.pr',
 426   'youtube.pt',
 427   'youtube.qa',
 428   'youtube.ro',
 429   'youtube.rs',
 430   'youtube.ru',
 431   'youtube.sa',
 432   'youtube.se',
 433   'youtube.sg',
 434   'youtube.si',
 435   'youtube.sk',
 436   'youtube.sn',
 437   'youtube.tn',
 438   'youtube.ua',
 439   'youtube.ug',
 440   'youtube.uy',
 441   'youtube.vn',
 442   'youtubeeducation.com',
 443   'youtubemobilesupport.com',
 444   'ytimg.com'
 445 )
 446
 447
 448 CC_HEADER = """// AUTOGENERATED FILE. DO NOT EDIT.
 449 //
 450 // (Update configs in components/domain_reliability/baked_in_configs and list
 451 // configs in components/domain_reliability/baked_in_configs.gypi instead.)
 452
 453 #include "components/domain_reliability/baked_in_configs.h"
 454
 455 #include <stdlib.h>
 456
 457 namespace domain_reliability {
 458
 459 const char* const kBakedInJsonConfigs[] = {
 460 """
 461
 462
 463 CC_FOOTER = """  NULL
 464 };
 465
 466 }  // namespace domain_reliability
 467 """
 468
 469
 470 def read_json_files_from_gypi(gypi_file):
 471   with open(gypi_file, 'r') as f:
 472     gypi_text = f.read()
 473   json_files = ast.literal_eval(gypi_text)['variables']['baked_in_configs']
 474   return json_files
 475
 476
 477 def domain_is_whitelisted(domain):
 478   return any(domain == e or domain.endswith('.' + e)  for e in DOMAIN_WHITELIST)
 479
 480
 481 def quote_and_wrap_text(text, width=79, prefix='  "', suffix='"'):
 482   max_length = width - len(prefix) - len(suffix)
 483   output = prefix
 484   line_length = 0
 485   for c in text:
 486     if c == "\"":
 487       c = "\\\""
 488     elif c == "\n":
 489       c = "\\n"
 490     elif c == "\\":
 491       c = "\\\\"
 492     if line_length + len(c) > max_length:
 493       output += suffix + "\n" + prefix
 494       line_length = 0
 495     output += c
 496     line_length += len(c)
 497   output += suffix
 498   return output
 499
 500
 501 def main():
 502   if len(sys.argv) != 4:
 503     print >> sys.stderr, (('Usage: %s <JSON pathname base directory> ' +
 504                            '<input .gypi file> <output .cpp file>') %
 505                           sys.argv[0])
 506     print >> sys.stderr, sys.modules[__name__].__doc__
 507     return 1
 508   json_path = sys.argv[1]
 509   gypi_file = sys.argv[2]
 510   cpp_file = sys.argv[3]
 511
 512   json_files = read_json_files_from_gypi(gypi_file)
 513   json_files = [ os.path.join(json_path, f) for f in json_files ]
 514   json_files = [ os.path.normpath(f) for f in json_files ]
 515
 516   cpp_code = CC_HEADER
 517   found_invalid_config = False
 518
 519   for json_file in json_files:
 520     with open(json_file, 'r') as f:
 521       json_text = f.read()
 522     try:
 523       config = json.loads(json_text)
 524     except ValueError, e:
 525       print >> sys.stderr, "%s: error parsing JSON: %s" % (json_file, e)
 526       found_invalid_config = True
 527       continue
 528     if 'monitored_domain' not in config:
 529       print >> sys.stderr, '%s: no monitored_domain found' % json_file
 530       found_invalid_config = True
 531       continue
 532     domain = config['monitored_domain']
 533     if not domain_is_whitelisted(domain):
 534       print >> sys.stderr, ('%s: monitored_domain "%s" not in whitelist' %
 535                             (json_file, domain))
 536       found_invalid_config = True
 537       continue
 538
 539     # Re-dump JSON to get a more compact representation.
 540     dumped_json_text = json.dumps(config, separators=(',', ':'))
 541
 542     cpp_code += "  // " + json_file + ":\n"
 543     cpp_code += quote_and_wrap_text(dumped_json_text) + ",\n"
 544     cpp_code += "\n"
 545
 546   cpp_code += CC_FOOTER
 547
 548   if found_invalid_config:
 549     return 1
 550
 551   with open(cpp_file, 'wb') as f:
 552     f.write(cpp_code)
 553
 554   return 0
 555
 556
 557 if __name__ == '__main__':
 558   sys.exit(main())