components/policy/tools/syntax_check_policy_template_json.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 '''
   7 Checks a policy_templates.json file for conformity to its syntax specification.
   8 '''
   9
  10 import json
  11 import optparse
  12 import os
  13 import re
  14 import sys
  15
  16
  17 LEADING_WHITESPACE = re.compile('^([ \t]*)')
  18 TRAILING_WHITESPACE = re.compile('.*?([ \t]+)$')
  19 # Matches all non-empty strings that contain no whitespaces.
  20 NO_WHITESPACE = re.compile('[^\s]+$')
  21
  22 # Convert a 'type' to the schema types it may be converted to.
  23 # The 'dict' type represents structured JSON data, and can be converted
  24 # to an 'object' or an 'array'.
  25 TYPE_TO_SCHEMA = {
  26   'int': [ 'integer' ],
  27   'list': [ 'array' ],
  28   'dict': [ 'object', 'array' ],
  29   'main': [ 'boolean' ],
  30   'string': [ 'string' ],
  31   'int-enum': [ 'integer' ],
  32   'string-enum': [ 'string' ],
  33   'string-enum-list': [ 'array' ],
  34   'external': [ 'object' ],
  35 }
  36
  37 # List of boolean policies that have been introduced with negative polarity in
  38 # the past and should not trigger the negative polarity check.
  39 LEGACY_INVERTED_POLARITY_WHITELIST = [
  40     'DeveloperToolsDisabled',
  41     'DeviceAutoUpdateDisabled',
  42     'Disable3DAPIs',
  43     'DisableAuthNegotiateCnameLookup',
  44     'DisablePluginFinder',
  45     'DisablePrintPreview',
  46     'DisableSafeBrowsingProceedAnyway',
  47     'DisableScreenshots',
  48     'DisableSpdy',
  49     'DisableSSLRecordSplitting',
  50     'DriveDisabled',
  51     'DriveDisabledOverCellular',
  52     'ExternalStorageDisabled',
  53     'SavingBrowserHistoryDisabled',
  54     'SyncDisabled',
  55 ]
  56
  57 class PolicyTemplateChecker(object):
  58
  59   def __init__(self):
  60     self.error_count = 0
  61     self.warning_count = 0
  62     self.num_policies = 0
  63     self.num_groups = 0
  64     self.num_policies_in_groups = 0
  65     self.options = None
  66     self.features = []
  67
  68   def _Error(self, message, parent_element=None, identifier=None,
  69              offending_snippet=None):
  70     self.error_count += 1
  71     error = ''
  72     if identifier is not None and parent_element is not None:
  73       error += 'In %s %s: ' % (parent_element, identifier)
  74     print error + 'Error: ' + message
  75     if offending_snippet is not None:
  76       print '  Offending:', json.dumps(offending_snippet, indent=2)
  77
  78   def _CheckContains(self, container, key, value_type,
  79                      optional=False,
  80                      parent_element='policy',
  81                      container_name=None,
  82                      identifier=None,
  83                      offending='__CONTAINER__',
  84                      regexp_check=None):
  85     '''
  86     Checks |container| for presence of |key| with value of type |value_type|.
  87     If |value_type| is string and |regexp_check| is specified, then an error is
  88     reported when the value does not match the regular expression object.
  89
  90     |value_type| can also be a list, if more than one type is supported.
  91
  92     The other parameters are needed to generate, if applicable, an appropriate
  93     human-readable error message of the following form:
  94
  95     In |parent_element| |identifier|:
  96       (if the key is not present):
  97       Error: |container_name| must have a |value_type| named |key|.
  98       Offending snippet: |offending| (if specified; defaults to |container|)
  99       (if the value does not have the required type):
 100       Error: Value of |key| must be a |value_type|.
 101       Offending snippet: |container[key]|
 102
 103     Returns: |container[key]| if the key is present, None otherwise.
 104     '''
 105     if identifier is None:
 106       try:
 107         identifier = container.get('name')
 108       except:
 109         self._Error('Cannot access container name of "%s".' % container_name)
 110         return None
 111     if container_name is None:
 112       container_name = parent_element
 113     if offending == '__CONTAINER__':
 114       offending = container
 115     if key not in container:
 116       if optional:
 117         return
 118       else:
 119         self._Error('%s must have a %s "%s".' %
 120                     (container_name.title(), value_type.__name__, key),
 121                     container_name, identifier, offending)
 122       return None
 123     value = container[key]
 124     value_types = value_type if isinstance(value_type, list) else [ value_type ]
 125     if not any(isinstance(value, type) for type in value_types):
 126       self._Error('Value of "%s" must be one of [ %s ].' %
 127                   (key, ', '.join([type.__name__ for type in value_types])),
 128                   container_name, identifier, value)
 129     if str in value_types and regexp_check and not regexp_check.match(value):
 130       self._Error('Value of "%s" must match "%s".' %
 131                   (key, regexp_check.pattern),
 132                   container_name, identifier, value)
 133     return value
 134
 135   def _AddPolicyID(self, id, policy_ids, policy):
 136     '''
 137     Adds |id| to |policy_ids|. Generates an error message if the
 138     |id| exists already; |policy| is needed for this message.
 139     '''
 140     if id in policy_ids:
 141       self._Error('Duplicate id', 'policy', policy.get('name'),
 142                   id)
 143     else:
 144       policy_ids.add(id)
 145
 146   def _CheckPolicyIDs(self, policy_ids):
 147     '''
 148     Checks a set of policy_ids to make sure it contains a continuous range
 149     of entries (i.e. no holes).
 150     Holes would not be a technical problem, but we want to ensure that nobody
 151     accidentally omits IDs.
 152     '''
 153     for i in range(len(policy_ids)):
 154       if (i + 1) not in policy_ids:
 155         self._Error('No policy with id: %s' % (i + 1))
 156
 157   def _CheckPolicySchema(self, policy, policy_type):
 158     '''Checks that the 'schema' field matches the 'type' field.'''
 159     self._CheckContains(policy, 'schema', dict)
 160     if isinstance(policy.get('schema'), dict):
 161       self._CheckContains(policy['schema'], 'type', str)
 162       schema_type = policy['schema'].get('type')
 163       if schema_type not in TYPE_TO_SCHEMA[policy_type]:
 164         self._Error('Schema type must match the existing type for policy %s' %
 165                     policy.get('name'))
 166
 167       # Checks that boolean policies are not negated (which makes them harder to
 168       # reason about).
 169       if (schema_type == 'boolean' and
 170           'disable' in policy.get('name').lower() and
 171           policy.get('name') not in LEGACY_INVERTED_POLARITY_WHITELIST):
 172         self._Error(('Boolean policy %s uses negative polarity, please make ' +
 173                      'new boolean policies follow the XYZEnabled pattern. ' +
 174                      'See also http://crbug.com/85687') % policy.get('name'))
 175
 176
 177   def _CheckPolicy(self, policy, is_in_group, policy_ids):
 178     if not isinstance(policy, dict):
 179       self._Error('Each policy must be a dictionary.', 'policy', None, policy)
 180       return
 181
 182     # There should not be any unknown keys in |policy|.
 183     for key in policy:
 184       if key not in ('name', 'type', 'caption', 'desc', 'device_only',
 185                      'supported_on', 'label', 'policies', 'items',
 186                      'example_value', 'features', 'deprecated', 'future',
 187                      'id', 'schema', 'max_size',
 188                      'default_for_enterprise_users'):
 189         self.warning_count += 1
 190         print ('In policy %s: Warning: Unknown key: %s' %
 191                (policy.get('name'), key))
 192
 193     # Each policy must have a name.
 194     self._CheckContains(policy, 'name', str, regexp_check=NO_WHITESPACE)
 195
 196     # Each policy must have a type.
 197     policy_types = ('group', 'main', 'string', 'int', 'list', 'int-enum',
 198                     'string-enum', 'string-enum-list', 'dict', 'external')
 199     policy_type = self._CheckContains(policy, 'type', str)
 200     if policy_type not in policy_types:
 201       self._Error('Policy type must be one of: ' + ', '.join(policy_types),
 202                   'policy', policy.get('name'), policy_type)
 203       return  # Can't continue for unsupported type.
 204
 205     # Each policy must have a caption message.
 206     self._CheckContains(policy, 'caption', str)
 207
 208     # Each policy must have a description message.
 209     self._CheckContains(policy, 'desc', str)
 210
 211     # If 'label' is present, it must be a string.
 212     self._CheckContains(policy, 'label', str, True)
 213
 214     # If 'deprecated' is present, it must be a bool.
 215     self._CheckContains(policy, 'deprecated', bool, True)
 216
 217     # If 'future' is present, it must be a bool.
 218     self._CheckContains(policy, 'future', bool, True)
 219
 220     if policy_type == 'group':
 221       # Groups must not be nested.
 222       if is_in_group:
 223         self._Error('Policy groups must not be nested.', 'policy', policy)
 224
 225       # Each policy group must have a list of policies.
 226       policies = self._CheckContains(policy, 'policies', list)
 227
 228       # Check sub-policies.
 229       if policies is not None:
 230         for nested_policy in policies:
 231           self._CheckPolicy(nested_policy, True, policy_ids)
 232
 233       # Groups must not have an |id|.
 234       if 'id' in policy:
 235         self._Error('Policies of type "group" must not have an "id" field.',
 236                     'policy', policy)
 237
 238       # Statistics.
 239       self.num_groups += 1
 240
 241     else:  # policy_type != group
 242       # Each policy must have a protobuf ID.
 243       id = self._CheckContains(policy, 'id', int)
 244       self._AddPolicyID(id, policy_ids, policy)
 245
 246       # 'schema' is the new 'type'.
 247       # TODO(joaodasilva): remove the 'type' checks once 'schema' is used
 248       # everywhere.
 249       self._CheckPolicySchema(policy, policy_type)
 250
 251       # Each policy must have a supported_on list.
 252       supported_on = self._CheckContains(policy, 'supported_on', list)
 253       if supported_on is not None:
 254         for s in supported_on:
 255           if not isinstance(s, str):
 256             self._Error('Entries in "supported_on" must be strings.', 'policy',
 257                         policy, supported_on)
 258
 259       # Each policy must have a 'features' dict.
 260       features = self._CheckContains(policy, 'features', dict)
 261
 262       # All the features must have a documenting message.
 263       if features:
 264         for feature in features:
 265           if not feature in self.features:
 266             self._Error('Unknown feature "%s". Known features must have a '
 267                         'documentation string in the messages dictionary.' %
 268                         feature, 'policy', policy.get('name', policy))
 269
 270       # All user policies must have a per_profile feature flag.
 271       if (not policy.get('device_only', False) and
 272           not policy.get('deprecated', False) and
 273           not filter(re.compile('^chrome_frame:.*').match, supported_on)):
 274         self._CheckContains(features, 'per_profile', bool,
 275                             container_name='features',
 276                             identifier=policy.get('name'))
 277
 278       # All policies must declare whether they allow changes at runtime.
 279       self._CheckContains(features, 'dynamic_refresh', bool,
 280                           container_name='features',
 281                           identifier=policy.get('name'))
 282
 283       # Each policy must have an 'example_value' of appropriate type.
 284       if policy_type == 'main':
 285         value_type = item_type = bool
 286       elif policy_type in ('string', 'string-enum'):
 287         value_type = item_type = str
 288       elif policy_type in ('int', 'int-enum'):
 289         value_type = item_type = int
 290       elif policy_type in ('list', 'string-enum-list'):
 291         value_type = list
 292         item_type = str
 293       elif policy_type == 'external':
 294         value_type = item_type = dict
 295       elif policy_type == 'dict':
 296         value_type = item_type = [ dict, list ]
 297       else:
 298         raise NotImplementedError('Unimplemented policy type: %s' % policy_type)
 299       self._CheckContains(policy, 'example_value', value_type)
 300
 301       # Statistics.
 302       self.num_policies += 1
 303       if is_in_group:
 304         self.num_policies_in_groups += 1
 305
 306     if policy_type in ('int-enum', 'string-enum', 'string-enum-list'):
 307       # Enums must contain a list of items.
 308       items = self._CheckContains(policy, 'items', list)
 309       if items is not None:
 310         if len(items) < 1:
 311           self._Error('"items" must not be empty.', 'policy', policy, items)
 312         for item in items:
 313           # Each item must have a name.
 314           # Note: |policy.get('name')| is used instead of |policy['name']|
 315           # because it returns None rather than failing when no key called
 316           # 'name' exists.
 317           self._CheckContains(item, 'name', str, container_name='item',
 318                               identifier=policy.get('name'),
 319                               regexp_check=NO_WHITESPACE)
 320
 321           # Each item must have a value of the correct type.
 322           self._CheckContains(item, 'value', item_type, container_name='item',
 323                               identifier=policy.get('name'))
 324
 325           # Each item must have a caption.
 326           self._CheckContains(item, 'caption', str, container_name='item',
 327                               identifier=policy.get('name'))
 328
 329     if policy_type == 'external':
 330       # Each policy referencing external data must specify a maximum data size.
 331       self._CheckContains(policy, 'max_size', int)
 332
 333   def _CheckMessage(self, key, value):
 334     # |key| must be a string, |value| a dict.
 335     if not isinstance(key, str):
 336       self._Error('Each message key must be a string.', 'message', key, key)
 337       return
 338
 339     if not isinstance(value, dict):
 340       self._Error('Each message must be a dictionary.', 'message', key, value)
 341       return
 342
 343     # Each message must have a desc.
 344     self._CheckContains(value, 'desc', str, parent_element='message',
 345                         identifier=key)
 346
 347     # Each message must have a text.
 348     self._CheckContains(value, 'text', str, parent_element='message',
 349                         identifier=key)
 350
 351     # There should not be any unknown keys in |value|.
 352     for vkey in value:
 353       if vkey not in ('desc', 'text'):
 354         self.warning_count += 1
 355         print 'In message %s: Warning: Unknown key: %s' % (key, vkey)
 356
 357   def _LeadingWhitespace(self, line):
 358     match = LEADING_WHITESPACE.match(line)
 359     if match:
 360       return match.group(1)
 361     return ''
 362
 363   def _TrailingWhitespace(self, line):
 364     match = TRAILING_WHITESPACE.match(line)
 365     if match:
 366       return match.group(1)
 367     return ''
 368
 369   def _LineError(self, message, line_number):
 370     self.error_count += 1
 371     print 'In line %d: Error: %s' % (line_number, message)
 372
 373   def _LineWarning(self, message, line_number):
 374     self.warning_count += 1
 375     print ('In line %d: Warning: Automatically fixing formatting: %s'
 376            % (line_number, message))
 377
 378   def _CheckFormat(self, filename):
 379     if self.options.fix:
 380       fixed_lines = []
 381     with open(filename) as f:
 382       indent = 0
 383       line_number = 0
 384       for line in f:
 385         line_number += 1
 386         line = line.rstrip('\n')
 387         # Check for trailing whitespace.
 388         trailing_whitespace = self._TrailingWhitespace(line)
 389         if len(trailing_whitespace) > 0:
 390           if self.options.fix:
 391             line = line.rstrip()
 392             self._LineWarning('Trailing whitespace.', line_number)
 393           else:
 394             self._LineError('Trailing whitespace.', line_number)
 395         if self.options.fix:
 396           if len(line) == 0:
 397             fixed_lines += ['\n']
 398             continue
 399         else:
 400           if line == trailing_whitespace:
 401             # This also catches the case of an empty line.
 402             continue
 403         # Check for correct amount of leading whitespace.
 404         leading_whitespace = self._LeadingWhitespace(line)
 405         if leading_whitespace.count('\t') > 0:
 406           if self.options.fix:
 407             leading_whitespace = leading_whitespace.replace('\t', '  ')
 408             line = leading_whitespace + line.lstrip()
 409             self._LineWarning('Tab character found.', line_number)
 410           else:
 411             self._LineError('Tab character found.', line_number)
 412         if line[len(leading_whitespace)] in (']', '}'):
 413           indent -= 2
 414         if line[0] != '#':  # Ignore 0-indented comments.
 415           if len(leading_whitespace) != indent:
 416             if self.options.fix:
 417               line = ' ' * indent + line.lstrip()
 418               self._LineWarning('Indentation should be ' + str(indent) +
 419                                 ' spaces.', line_number)
 420             else:
 421               self._LineError('Bad indentation. Should be ' + str(indent) +
 422                               ' spaces.', line_number)
 423         if line[-1] in ('[', '{'):
 424           indent += 2
 425         if self.options.fix:
 426           fixed_lines.append(line + '\n')
 427
 428     # If --fix is specified: backup the file (deleting any existing backup),
 429     # then write the fixed version with the old filename.
 430     if self.options.fix:
 431       if self.options.backup:
 432         backupfilename = filename + '.bak'
 433         if os.path.exists(backupfilename):
 434           os.remove(backupfilename)
 435         os.rename(filename, backupfilename)
 436       with open(filename, 'w') as f:
 437         f.writelines(fixed_lines)
 438
 439   def Main(self, filename, options):
 440     try:
 441       with open(filename) as f:
 442         data = eval(f.read())
 443     except:
 444       import traceback
 445       traceback.print_exc(file=sys.stdout)
 446       self._Error('Invalid Python/JSON syntax.')
 447       return 1
 448     if data == None:
 449       self._Error('Invalid Python/JSON syntax.')
 450       return 1
 451     self.options = options
 452
 453     # First part: check JSON structure.
 454
 455     # Check (non-policy-specific) message definitions.
 456     messages = self._CheckContains(data, 'messages', dict,
 457                                    parent_element=None,
 458                                    container_name='The root element',
 459                                    offending=None)
 460     if messages is not None:
 461       for message in messages:
 462         self._CheckMessage(message, messages[message])
 463         if message.startswith('doc_feature_'):
 464           self.features.append(message[12:])
 465
 466     # Check policy definitions.
 467     policy_definitions = self._CheckContains(data, 'policy_definitions', list,
 468                                              parent_element=None,
 469                                              container_name='The root element',
 470                                              offending=None)
 471     if policy_definitions is not None:
 472       policy_ids = set()
 473       for policy in policy_definitions:
 474         self._CheckPolicy(policy, False, policy_ids)
 475       self._CheckPolicyIDs(policy_ids)
 476
 477     # Second part: check formatting.
 478     self._CheckFormat(filename)
 479
 480     # Third part: summary and exit.
 481     print ('Finished checking %s. %d errors, %d warnings.' %
 482         (filename, self.error_count, self.warning_count))
 483     if self.options.stats:
 484       if self.num_groups > 0:
 485         print ('%d policies, %d of those in %d groups (containing on '
 486                'average %.1f policies).' %
 487                (self.num_policies, self.num_policies_in_groups, self.num_groups,
 488                  (1.0 * self.num_policies_in_groups / self.num_groups)))
 489       else:
 490         print self.num_policies, 'policies, 0 policy groups.'
 491     if self.error_count > 0:
 492       return 1
 493     return 0
 494
 495   def Run(self, argv, filename=None):
 496     parser = optparse.OptionParser(
 497         usage='usage: %prog [options] filename',
 498         description='Syntax check a policy_templates.json file.')
 499     parser.add_option('--fix', action='store_true',
 500                       help='Automatically fix formatting.')
 501     parser.add_option('--backup', action='store_true',
 502                       help='Create backup of original file (before fixing).')
 503     parser.add_option('--stats', action='store_true',
 504                       help='Generate statistics.')
 505     (options, args) = parser.parse_args(argv)
 506     if filename is None:
 507       if len(args) != 2:
 508         parser.print_help()
 509         sys.exit(1)
 510       filename = args[1]
 511     return self.Main(filename, options)
 512
 513
 514 if __name__ == '__main__':
 515   sys.exit(PolicyTemplateChecker().Run(sys.argv))