toolkit/components/telemetry/build_scripts/gen_histogram_data.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Write out histogram information for C++.  The histograms are defined
   6 # in a file provided as a command-line argument.
   7
   8 import sys
   9 from os import path
  10
  11 import buildconfig
  12 from mozparsers import parse_histograms
  13 from mozparsers.shared_telemetry_utils import ParserError, static_assert
  14
  15 COMPONENTS_PATH = path.abspath(
  16     path.join(path.dirname(__file__), path.pardir, path.pardir)
  17 )
  18 sys.path.append(
  19     path.join(COMPONENTS_PATH, "glean", "build_scripts", "glean_parser_ext")
  20 )
  21 from string_table import StringTable
  22
  23 banner = """/* This file is auto-generated, see gen_histogram_data.py.  */
  24 """
  25
  26
  27 def print_array_entry(
  28     output,
  29     histogram,
  30     name_index,
  31     exp_index,
  32     label_index,
  33     label_count,
  34     key_index,
  35     key_count,
  36     store_index,
  37     store_count,
  38 ):
  39     if histogram.record_on_os(buildconfig.substs["OS_TARGET"]):
  40         print(
  41             "  { %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %s, %s, %s, %s, %s, %s },"
  42             % (
  43                 histogram.low(),
  44                 histogram.high(),
  45                 histogram.n_buckets(),
  46                 name_index,
  47                 exp_index,
  48                 label_count,
  49                 key_count,
  50                 store_count,
  51                 label_index,
  52                 key_index,
  53                 store_index,
  54                 " | ".join(histogram.record_in_processes_enum()),
  55                 "true" if histogram.keyed() else "false",
  56                 histogram.nsITelemetry_kind(),
  57                 histogram.dataset(),
  58                 " | ".join(histogram.products_enum()),
  59             ),
  60             file=output,
  61         )
  62
  63
  64 def write_histogram_table(output, histograms):
  65     string_table = StringTable()
  66
  67     label_table = []
  68     label_count = 0
  69     keys_table = []
  70     keys_count = 0
  71     store_table = []
  72     total_store_count = 0
  73
  74     print("constexpr HistogramInfo gHistogramInfos[] = {", file=output)
  75     for histogram in histograms:
  76         name_index = string_table.stringIndex(histogram.name())
  77         exp_index = string_table.stringIndex(histogram.expiration())
  78
  79         labels = histogram.labels()
  80         label_index = 0
  81         if len(labels) > 0:
  82             label_index = label_count
  83             label_table.append((histogram.name(), string_table.stringIndexes(labels)))
  84             label_count += len(labels)
  85
  86         keys = histogram.keys()
  87         key_index = 0
  88         if len(keys) > 0:
  89             key_index = keys_count
  90             keys_table.append((histogram.name(), string_table.stringIndexes(keys)))
  91             keys_count += len(keys)
  92
  93         stores = histogram.record_into_store()
  94         store_index = 0
  95         if stores == ["main"]:
  96             # if count == 1 && offset == UINT16_MAX -> only main store
  97             store_index = "UINT16_MAX"
  98         else:
  99             store_index = total_store_count
 100             store_table.append((histogram.name(), string_table.stringIndexes(stores)))
 101             total_store_count += len(stores)
 102
 103         print_array_entry(
 104             output,
 105             histogram,
 106             name_index,
 107             exp_index,
 108             label_index,
 109             len(labels),
 110             key_index,
 111             len(keys),
 112             store_index,
 113             len(stores),
 114         )
 115     print("};\n", file=output)
 116
 117     strtab_name = "gHistogramStringTable"
 118     string_table.writeDefinition(output, strtab_name)
 119     static_assert(output, "sizeof(%s) <= UINT32_MAX" % strtab_name, "index overflow")
 120
 121     print("\n#if defined(_MSC_VER) && !defined(__clang__)", file=output)
 122     print("const uint32_t gHistogramLabelTable[] = {", file=output)
 123     print("#else", file=output)
 124     print("constexpr uint32_t gHistogramLabelTable[] = {", file=output)
 125     print("#endif", file=output)
 126     for name, indexes in label_table:
 127         print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
 128     print("};", file=output)
 129     static_assert(
 130         output, "sizeof(gHistogramLabelTable) <= UINT16_MAX", "index overflow"
 131     )
 132
 133     print("\n#if defined(_MSC_VER) && !defined(__clang__)", file=output)
 134     print("const uint32_t gHistogramKeyTable[] = {", file=output)
 135     print("#else", file=output)
 136     print("constexpr uint32_t gHistogramKeyTable[] = {", file=output)
 137     print("#endif", file=output)
 138     for name, indexes in keys_table:
 139         print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
 140     print("};", file=output)
 141     static_assert(output, "sizeof(gHistogramKeyTable) <= UINT16_MAX", "index overflow")
 142
 143     store_table_name = "gHistogramStoresTable"
 144     print("\n#if defined(_MSC_VER) && !defined(__clang__)", file=output)
 145     print("const uint32_t {}[] = {{".format(store_table_name), file=output)
 146     print("#else", file=output)
 147     print("constexpr uint32_t {}[] = {{".format(store_table_name), file=output)
 148     print("#endif", file=output)
 149     for name, indexes in store_table:
 150         print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
 151     print("};", file=output)
 152     static_assert(
 153         output, "sizeof(%s) <= UINT16_MAX" % store_table_name, "index overflow"
 154     )
 155
 156
 157 # Write out static asserts for histogram data.  We'd prefer to perform
 158 # these checks in this script itself, but since several histograms
 159 # (generally enumerated histograms) use compile-time constants for
 160 # their upper bounds, we have to let the compiler do the checking.
 161
 162
 163 def static_asserts_for_boolean(output, histogram):
 164     pass
 165
 166
 167 def static_asserts_for_flag(output, histogram):
 168     pass
 169
 170
 171 def static_asserts_for_count(output, histogram):
 172     pass
 173
 174
 175 def static_asserts_for_enumerated(output, histogram):
 176     n_values = histogram.high()
 177     static_assert(
 178         output, "%s > 2" % n_values, "Not enough values for %s" % histogram.name()
 179     )
 180
 181
 182 def shared_static_asserts(output, histogram):
 183     name = histogram.name()
 184     low = histogram.low()
 185     high = histogram.high()
 186     n_buckets = histogram.n_buckets()
 187     static_assert(output, "%s < %s" % (low, high), "low >= high for %s" % name)
 188     static_assert(output, "%s > 2" % n_buckets, "Not enough values for %s" % name)
 189     static_assert(output, "%s >= 1" % low, "Incorrect low value for %s" % name)
 190     static_assert(
 191         output,
 192         "%s > %s" % (high, n_buckets),
 193         "high must be > number of buckets for %s;"
 194         " you may want an enumerated histogram" % name,
 195     )
 196
 197
 198 def static_asserts_for_linear(output, histogram):
 199     shared_static_asserts(output, histogram)
 200
 201
 202 def static_asserts_for_exponential(output, histogram):
 203     shared_static_asserts(output, histogram)
 204
 205
 206 def write_histogram_static_asserts(output, histograms):
 207     print(
 208         """
 209 // Perform the checks at the beginning of HistogramGet at
 210 // compile time, so that incorrect histogram definitions
 211 // give compile-time errors, not runtime errors.""",
 212         file=output,
 213     )
 214
 215     table = {
 216         "boolean": static_asserts_for_boolean,
 217         "flag": static_asserts_for_flag,
 218         "count": static_asserts_for_count,
 219         "enumerated": static_asserts_for_enumerated,
 220         "categorical": static_asserts_for_enumerated,
 221         "linear": static_asserts_for_linear,
 222         "exponential": static_asserts_for_exponential,
 223     }
 224
 225     target_os = buildconfig.substs["OS_TARGET"]
 226     for histogram in histograms:
 227         kind = histogram.kind()
 228         if not histogram.record_on_os(target_os):
 229             continue
 230
 231         if kind not in table:
 232             raise Exception(
 233                 'Unknown kind "%s" for histogram "%s".' % (kind, histogram.name())
 234             )
 235         fn = table[kind]
 236         fn(output, histogram)
 237
 238
 239 def write_histogram_ranges(output, histograms):
 240     # This generates static data to avoid costly initialization of histograms
 241     # (especially exponential ones which require log and exp calls) at runtime.
 242     # The format must exactly match that required in histogram.cc, which is
 243     # 0, buckets..., INT_MAX. Additionally, the list ends in a 0 to aid asserts
 244     # that validate that the length of the ranges list is correct.U cache miss.
 245     print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
 246     print("const int gHistogramBucketLowerBounds[] = {", file=output)
 247     print("#else", file=output)
 248     print("constexpr int gHistogramBucketLowerBounds[] = {", file=output)
 249     print("#endif", file=output)
 250
 251     # Print the dummy buckets for expired histograms, and set the offset to match.
 252     print("0,1,2,INT_MAX,", file=output)
 253     offset = 4
 254     ranges_offsets = {}
 255
 256     for histogram in histograms:
 257         ranges = tuple(histogram.ranges())
 258         if ranges not in ranges_offsets:
 259             ranges_offsets[ranges] = offset
 260             # Suffix each ranges listing with INT_MAX, to match histogram.cc's
 261             # expected format.
 262             offset += len(ranges) + 1
 263             print(",".join(map(str, ranges)), ",INT_MAX,", file=output)
 264     print("0};", file=output)
 265
 266     if offset > 32767:
 267         raise Exception("Histogram offsets exceeded maximum value for an int16_t.")
 268
 269     target_os = buildconfig.substs["OS_TARGET"]
 270     print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
 271     print("const int16_t gHistogramBucketLowerBoundIndex[] = {", file=output)
 272     print("#else", file=output)
 273     print("constexpr int16_t gHistogramBucketLowerBoundIndex[] = {", file=output)
 274     print("#endif", file=output)
 275     for histogram in histograms:
 276         if histogram.record_on_os(target_os):
 277             our_offset = ranges_offsets[tuple(histogram.ranges())]
 278             print("%d," % our_offset, file=output)
 279
 280     print("};", file=output)
 281
 282
 283 def main(output, *filenames):
 284     try:
 285         histograms = list(parse_histograms.from_files(filenames))
 286     except ParserError as ex:
 287         print("\nError processing histograms:\n" + str(ex) + "\n")
 288         sys.exit(1)
 289
 290     print(banner, file=output)
 291     write_histogram_table(output, histograms)
 292     write_histogram_ranges(output, histograms)
 293     write_histogram_static_asserts(output, histograms)
 294
 295
 296 if __name__ == "__main__":
 297     main(sys.stdout, *sys.argv[1:])