toolkit/components/telemetry/build_scripts/gen_event_data.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Write out event information for C++. The events are defined
   6 # in a file provided as a command-line argument.
   7
   8 import itertools
   9 import json
  10 import sys
  11 from collections import OrderedDict
  12 from os import path
  13
  14 from mozparsers import parse_events
  15 from mozparsers.shared_telemetry_utils import ParserError, static_assert
  16
  17 COMPONENTS_PATH = path.abspath(
  18     path.join(path.dirname(__file__), path.pardir, path.pardir)
  19 )
  20 sys.path.append(
  21     path.join(COMPONENTS_PATH, "glean", "build_scripts", "glean_parser_ext")
  22 )
  23 from string_table import StringTable
  24
  25 # The banner/text at the top of the generated file.
  26 banner = """/* This file is auto-generated, only for internal use in TelemetryEvent.h,
  27    see gen_event_data.py. */
  28 """
  29
  30 file_header = """\
  31 #ifndef mozilla_TelemetryEventData_h
  32 #define mozilla_TelemetryEventData_h
  33 #include "core/EventInfo.h"
  34 #include "nsITelemetry.h"
  35 namespace {
  36 """
  37
  38 file_footer = """\
  39 } // namespace
  40 #endif // mozilla_TelemetryEventData_h
  41 """
  42
  43
  44 def write_extra_table(events, output, string_table):
  45     table_name = "gExtraKeysTable"
  46     extra_table = []
  47     extra_count = 0
  48
  49     print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
  50     print("const uint32_t %s[] = {" % table_name, file=output)
  51     print("#else", file=output)
  52     print("constexpr uint32_t %s[] = {" % table_name, file=output)
  53     print("#endif", file=output)
  54
  55     for e in events:
  56         extra_index = 0
  57         extra_keys = e.extra_keys
  58         if len(extra_keys) > 0:
  59             extra_index = extra_count
  60             extra_count += len(extra_keys)
  61             indexes = string_table.stringIndexes(extra_keys)
  62
  63             print(
  64                 "  // %s, [%s], [%s]"
  65                 % (e.category, ", ".join(e.methods), ", ".join(e.objects)),
  66                 file=output,
  67             )
  68             print("  // extra_keys: %s" % ", ".join(extra_keys), file=output)
  69             print("  %s," % ", ".join(map(str, indexes)), file=output)
  70
  71         extra_table.append((extra_index, len(extra_keys)))
  72
  73     print("};", file=output)
  74     static_assert(output, "sizeof(%s) <= UINT32_MAX" % table_name, "index overflow")
  75
  76     return extra_table
  77
  78
  79 def write_common_event_table(events, output, string_table, extra_table):
  80     table_name = "gCommonEventInfo"
  81
  82     print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
  83     print("const CommonEventInfo %s[] = {" % table_name, file=output)
  84     print("#else", file=output)
  85     print("constexpr CommonEventInfo %s[] = {" % table_name, file=output)
  86     print("#endif", file=output)
  87
  88     for e, extras in zip(events, extra_table):
  89         # Write a comment to make the file human-readable.
  90         print("  // category: %s" % e.category, file=output)
  91         print("  // methods: [%s]" % ", ".join(e.methods), file=output)
  92         print("  // objects: [%s]" % ", ".join(e.objects), file=output)
  93
  94         # Write the common info structure
  95         print(
  96             "  {%d, %d, %d, %d, %s, %s, %s },"
  97             % (
  98                 string_table.stringIndex(e.category),
  99                 string_table.stringIndex(e.expiry_version),
 100                 extras[0],  # extra keys index
 101                 extras[1],  # extra keys count
 102                 e.dataset,
 103                 " | ".join(e.record_in_processes_enum),
 104                 " | ".join(e.products_enum),
 105             ),
 106             file=output,
 107         )
 108
 109     print("};", file=output)
 110     static_assert(output, "sizeof(%s) <= UINT32_MAX" % table_name, "index overflow")
 111
 112
 113 def write_event_table(events, output, string_table):
 114     table_name = "gEventInfo"
 115
 116     print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
 117     print("const EventInfo %s[] = {" % table_name, file=output)
 118     print("#else", file=output)
 119     print("constexpr EventInfo %s[] = {" % table_name, file=output)
 120     print("#endif", file=output)
 121
 122     for common_info_index, e in enumerate(events):
 123         for method_name, object_name in itertools.product(e.methods, e.objects):
 124             print(
 125                 "  // category: %s, method: %s, object: %s"
 126                 % (e.category, method_name, object_name),
 127                 file=output,
 128             )
 129
 130             print(
 131                 "  {gCommonEventInfo[%d], %d, %d},"
 132                 % (
 133                     common_info_index,
 134                     string_table.stringIndex(method_name),
 135                     string_table.stringIndex(object_name),
 136                 ),
 137                 file=output,
 138             )
 139
 140     print("};", file=output)
 141     static_assert(output, "sizeof(%s) <= UINT32_MAX" % table_name, "index overflow")
 142
 143
 144 def generate_JSON_definitions(output, *filenames):
 145     """Write the event definitions to a JSON file.
 146
 147     :param output: the file to write the content to.
 148     :param filenames: a list of filenames provided by the build system.
 149            We only support a single file.
 150     """
 151     # Load the event data.
 152     events = []
 153     for filename in filenames:
 154         try:
 155             batch = parse_events.load_events(filename, True)
 156             events.extend(batch)
 157         except ParserError as ex:
 158             print("\nError processing %s:\n%s\n" % (filename, str(ex)), file=sys.stderr)
 159             sys.exit(1)
 160
 161     event_definitions = OrderedDict()
 162     for event in events:
 163         category = event.category
 164
 165         if category not in event_definitions:
 166             event_definitions[category] = OrderedDict()
 167
 168         event_definitions[category][event.name] = OrderedDict(
 169             {
 170                 "methods": event.methods,
 171                 "objects": event.objects,
 172                 "extra_keys": event.extra_keys,
 173                 "record_on_release": (
 174                     True if event.dataset_short == "opt-out" else False
 175                 ),
 176                 # We don't expire dynamic-builtin scalars: they're only meant for
 177                 # use in local developer builds anyway. They will expire when rebuilding.
 178                 "expires": event.expiry_version,
 179                 "expired": False,
 180                 "products": event.products,
 181             }
 182         )
 183
 184     json.dump(event_definitions, output, sort_keys=True)
 185
 186
 187 def main(output, *filenames):
 188     # Load the event data.
 189     events = []
 190     for filename in filenames:
 191         try:
 192             batch = parse_events.load_events(filename, True)
 193             events.extend(batch)
 194         except ParserError as ex:
 195             print("\nError processing %s:\n%s\n" % (filename, str(ex)), file=sys.stderr)
 196             sys.exit(1)
 197
 198     # Write the scalar data file.
 199     print(banner, file=output)
 200     print(file_header, file=output)
 201
 202     # Write the extra keys table.
 203     string_table = StringTable()
 204     extra_table = write_extra_table(events, output, string_table)
 205     print("", file=output)
 206
 207     # Write a table with the common event data.
 208     write_common_event_table(events, output, string_table, extra_table)
 209     print("", file=output)
 210
 211     # Write the data for individual events.
 212     write_event_table(events, output, string_table)
 213     print("", file=output)
 214
 215     # Write the string table.
 216     string_table_name = "gEventsStringTable"
 217     string_table.writeDefinition(output, string_table_name)
 218     static_assert(
 219         output, "sizeof(%s) <= UINT32_MAX" % string_table_name, "index overflow"
 220     )
 221     print("", file=output)
 222
 223     print(file_footer, file=output)
 224
 225
 226 if __name__ == "__main__":
 227     main(sys.stdout, *sys.argv[1:])