chrome/common/extensions/docs/server2/schema_processor.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 from collections import defaultdict, Mapping
   6 import traceback
   7
   8 from third_party.json_schema_compiler import json_parse, idl_schema, idl_parser
   9 from reference_resolver import ReferenceResolver
  10 from compiled_file_system import CompiledFileSystem
  11
  12 class SchemaProcessorForTest(object):
  13   '''Fake SchemaProcessor class. Returns the original schema, without
  14   processing.
  15   '''
  16   def Process(self, path, file_data):
  17     if path.endswith('.idl'):
  18       idl = idl_schema.IDLSchema(idl_parser.IDLParser().ParseData(file_data))
  19       # Wrap the result in a list so that it behaves like JSON API data.
  20       return [idl.process()[0]]
  21     return json_parse.Parse(file_data)
  22
  23 class SchemaProcessorFactoryForTest(object):
  24   '''Returns a fake SchemaProcessor class to be used for testing.
  25   '''
  26   def Create(self, retain_inlined_types):
  27     return SchemaProcessorForTest()
  28
  29
  30 class SchemaProcessorFactory(object):
  31   '''Factory for creating the schema processing utility.
  32   '''
  33   def __init__(self,
  34                reference_resolver,
  35                api_models,
  36                features_bundle,
  37                compiled_fs_factory,
  38                file_system):
  39     self._reference_resolver = reference_resolver
  40     self._api_models = api_models
  41     self._features_bundle = features_bundle
  42     self._compiled_fs_factory = compiled_fs_factory
  43     self._file_system = file_system
  44
  45   def Create(self, retain_inlined_types):
  46     return SchemaProcessor(self._reference_resolver.Get(),
  47                            self._api_models.Get(),
  48                            self._features_bundle.Get(),
  49                            self._compiled_fs_factory,
  50                            self._file_system,
  51                            retain_inlined_types)
  52
  53
  54 class SchemaProcessor(object):
  55   '''Helper for parsing the API schema.
  56   '''
  57   def __init__(self,
  58                reference_resolver,
  59                api_models,
  60                features_bundle,
  61                compiled_fs_factory,
  62                file_system,
  63                retain_inlined_types):
  64     self._reference_resolver = reference_resolver
  65     self._api_models = api_models
  66     self._features_bundle = features_bundle
  67     self._retain_inlined_types = retain_inlined_types
  68     self._compiled_file_system = compiled_fs_factory.Create(
  69         file_system, self.Process, SchemaProcessor, category='json-cache')
  70     self._api_stack = []
  71
  72   def _RemoveNoDocs(self, item):
  73     '''Removes nodes that should not be rendered from an API schema.
  74     '''
  75     if json_parse.IsDict(item):
  76       if item.get('nodoc', False):
  77         return True
  78       for key, value in item.items():
  79         if self._RemoveNoDocs(value):
  80           del item[key]
  81     elif type(item) == list:
  82       to_remove = []
  83       for i in item:
  84         if self._RemoveNoDocs(i):
  85           to_remove.append(i)
  86       for i in to_remove:
  87         item.remove(i)
  88     return False
  89
  90
  91   def _DetectInlineableTypes(self, schema):
  92     '''Look for documents that are only referenced once and mark them as inline.
  93     Actual inlining is done by _InlineDocs.
  94     '''
  95     if not schema.get('types'):
  96       return
  97
  98     ignore = frozenset(('value', 'choices'))
  99     refcounts = defaultdict(int)
 100     # Use an explicit stack instead of recursion.
 101     stack = [schema]
 102
 103     while stack:
 104       node = stack.pop()
 105       if isinstance(node, list):
 106         stack.extend(node)
 107       elif isinstance(node, Mapping):
 108         if '$ref' in node:
 109           refcounts[node['$ref']] += 1
 110         stack.extend(v for k, v in node.iteritems() if k not in ignore)
 111
 112     for type_ in schema['types']:
 113       if not 'noinline_doc' in type_:
 114         if refcounts[type_['id']] == 1:
 115           type_['inline_doc'] = True
 116
 117
 118   def _InlineDocs(self, schema):
 119     '''Replace '$ref's that refer to inline_docs with the json for those docs.
 120     If |retain_inlined_types| is False, then the inlined nodes are removed
 121     from the schema.
 122     '''
 123     inline_docs = {}
 124     types_without_inline_doc = []
 125     internal_api = False
 126
 127     api_features = self._features_bundle.GetAPIFeatures().Get()
 128     # We don't want to inline the events API, as it's handled differently
 129     # Also, the webviewTag API is handled differently, as it only exists
 130     # for the purpose of documentation, it's not a true internal api
 131     namespace = schema.get('namespace', '')
 132     if namespace != 'events' and namespace != 'webviewTag':
 133       internal_api = api_features.get(schema.get('namespace', ''), {}).get(
 134           'internal', False)
 135
 136     api_refs = set()
 137     # Gather refs to internal APIs
 138     def gather_api_refs(node):
 139       if isinstance(node, list):
 140         for i in node:
 141           gather_api_refs(i)
 142       elif isinstance(node, Mapping):
 143         ref = node.get('$ref')
 144         if ref:
 145           api_refs.add(ref)
 146         for k, v in node.iteritems():
 147           gather_api_refs(v)
 148     gather_api_refs(schema)
 149
 150     if len(api_refs) > 0:
 151       api_list = self._api_models.GetNames()
 152       api_name = schema.get('namespace', '')
 153       self._api_stack.append(api_name)
 154       for api in self._api_stack:
 155         if api in api_list:
 156           api_list.remove(api)
 157       for ref in api_refs:
 158         model, node_info = self._reference_resolver.GetRefModel(ref, api_list)
 159         if model and api_features.get(model.name, {}).get('internal', False):
 160           category, name = node_info
 161           for ref_schema in self._compiled_file_system.GetFromFile(
 162               model.source_file).Get():
 163             if category == 'type':
 164               for type_json in ref_schema.get('types'):
 165                 if type_json['id'] == name:
 166                   inline_docs[ref] = type_json
 167             elif category == 'event':
 168               for type_json in ref_schema.get('events'):
 169                 if type_json['name'] == name:
 170                   inline_docs[ref] = type_json
 171       self._api_stack.remove(api_name)
 172
 173     types = schema.get('types')
 174     if types:
 175       # Gather the types with inline_doc.
 176       for type_ in types:
 177         if type_.get('inline_doc'):
 178           inline_docs[type_['id']] = type_
 179           if not self._retain_inlined_types:
 180             for k in ('description', 'id', 'inline_doc'):
 181               type_.pop(k, None)
 182         elif internal_api:
 183           inline_docs[type_['id']] = type_
 184           # For internal apis that are not inline_doc we want to retain them
 185           # in the schema (i.e. same behaviour as remain_inlined_types)
 186           types_without_inline_doc.append(type_)
 187         else:
 188           types_without_inline_doc.append(type_)
 189       if not self._retain_inlined_types:
 190         schema['types'] = types_without_inline_doc
 191
 192     def apply_inline(node):
 193       if isinstance(node, list):
 194         for i in node:
 195           apply_inline(i)
 196       elif isinstance(node, Mapping):
 197         ref = node.get('$ref')
 198         if ref and ref in inline_docs:
 199           node.update(inline_docs[ref])
 200           del node['$ref']
 201         for k, v in node.iteritems():
 202           apply_inline(v)
 203
 204     apply_inline(schema)
 205
 206
 207   def Process(self, path, file_data):
 208     '''Parses |file_data| using a method determined by checking the
 209     extension of the file at the given |path|. Then, trims 'nodoc' and if
 210     |self.retain_inlined_types| is given and False, removes inlineable types
 211     from the parsed schema data.
 212     '''
 213     def trim_and_inline(schema, is_idl=False):
 214       '''Modifies an API schema in place by removing nodes that shouldn't be
 215       documented and inlining schema types that are only referenced once.
 216       '''
 217       if self._RemoveNoDocs(schema):
 218         # A return of True signifies that the entire schema should not be
 219         # documented. Otherwise, only nodes that request 'nodoc' are removed.
 220         return None
 221       if is_idl:
 222         self._DetectInlineableTypes(schema)
 223       self._InlineDocs(schema)
 224       return schema
 225
 226     if path.endswith('.idl'):
 227       idl = idl_schema.IDLSchema(
 228           idl_parser.IDLParser().ParseData(file_data))
 229       # Wrap the result in a list so that it behaves like JSON API data.
 230       return [trim_and_inline(idl.process()[0], is_idl=True)]
 231
 232     try:
 233       schemas = json_parse.Parse(file_data)
 234     except:
 235       raise ValueError('Cannot parse "%s" as JSON:\n%s' %
 236                        (path, traceback.format_exc()))
 237     for schema in schemas:
 238       # Schemas could consist of one API schema (data for a specific API file)
 239       # or multiple (data from extension_api.json).
 240       trim_and_inline(schema)
 241     return schemas