upb_generator/minitable/generator.cc

   1 // Protocol Buffers - Google's data interrdchange format
   2 // Copyright 2023 Google LLC.  All rights reserved.
   3 //
   4 // Use of this source code is governed by a BSD-style
   5 // license that can be found in the LICENSE file or at
   6 // https://developers.google.com/open-source/licenses/bsd
   7
   8 #include "upb_generator/minitable/generator.h"
   9
  10 #include <cstdint>
  11 #include <map>
  12 #include <string>
  13 #include <utility>
  14 #include <vector>
  15
  16 #include "absl/container/flat_hash_set.h"
  17 #include "absl/log/absl_check.h"
  18 #include "absl/strings/str_cat.h"
  19 #include "absl/strings/string_view.h"
  20 #include "absl/strings/substitute.h"
  21 #include "upb/mini_table/enum.h"
  22 #include "upb/mini_table/field.h"
  23 #include "upb/mini_table/internal/field.h"
  24 #include "upb/mini_table/message.h"
  25 #include "upb/reflection/def.hpp"
  26 #include "upb_generator/common.h"
  27 #include "upb_generator/common/names.h"
  28 #include "upb_generator/file_layout.h"
  29 #include "upb_generator/minitable/fasttable.h"
  30 #include "upb_generator/minitable/names.h"
  31 #include "upb_generator/minitable/names_internal.h"
  32 #include "upb_generator/plugin.h"
  33
  34 // Must be last.
  35 #include "upb/port/def.inc"
  36
  37 namespace upb {
  38 namespace generator {
  39 namespace {
  40
  41 // Some local convenience aliases for MiniTable variable names.
  42
  43 std::string MessageVarName(upb::MessageDefPtr message) {
  44   return MiniTableMessageVarName(message.full_name());
  45 }
  46
  47 std::string MessagePtrVarName(upb::MessageDefPtr message) {
  48   return MiniTableMessagePtrVarName(message.full_name());
  49 }
  50
  51 std::string EnumVarName(upb::EnumDefPtr e) {
  52   return MiniTableEnumVarName(e.full_name());
  53 }
  54
  55 std::string ExtensionVarName(upb::FieldDefPtr ext) {
  56   return MiniTableExtensionVarName(ext.full_name());
  57 }
  58
  59 std::string FileVarName(upb::FileDefPtr file) {
  60   return MiniTableFileVarName(file.name());
  61 }
  62
  63 std::string HeaderFilename(upb::FileDefPtr file, bool bootstrap) {
  64   return MiniTableHeaderFilename(file.name(), bootstrap);
  65 }
  66
  67 std::string ArchDependentSize(int64_t size32, int64_t size64) {
  68   if (size32 == size64) return absl::StrCat(size32);
  69   return absl::Substitute("UPB_SIZE($0, $1)", size32, size64);
  70 }
  71
  72 std::string FieldInitializer(const DefPoolPair& pools, upb::FieldDefPtr field) {
  73   return upb::generator::FieldInitializer(field, pools.GetField64(field),
  74                                           pools.GetField32(field));
  75 }
  76
  77 // Writes a single field into a .upb.c source file.
  78 void WriteMessageField(upb::FieldDefPtr field,
  79                        const upb_MiniTableField* field64,
  80                        const upb_MiniTableField* field32, Output& output) {
  81   output("  $0,\n", upb::generator::FieldInitializer(field, field64, field32));
  82 }
  83
  84 std::string GetSub(upb::FieldDefPtr field, bool is_extension) {
  85   if (auto message_def = field.message_type()) {
  86     return absl::Substitute("{.UPB_PRIVATE(submsg) = &$0}",
  87                             is_extension ? MessageVarName(message_def)
  88                                          : MessagePtrVarName(message_def));
  89   }
  90
  91   if (auto enum_def = field.enum_subdef()) {
  92     if (enum_def.is_closed()) {
  93       return absl::Substitute("{.UPB_PRIVATE(subenum) = &$0}",
  94                               MiniTableEnumVarName(enum_def.full_name()));
  95     }
  96   }
  97
  98   return std::string("{.UPB_PRIVATE(submsg) = NULL}");
  99 }
 100
 101 bool IsCrossFile(upb::FieldDefPtr field) {
 102   return field.message_type() != field.containing_type();
 103 }
 104
 105 // Writes a single message into a .upb.c source file.
 106 void WriteMessage(upb::MessageDefPtr message, const DefPoolPair& pools,
 107                   const MiniTableOptions& options, Output& output) {
 108   std::string fields_array_ref = "NULL";
 109   std::string submsgs_array_ref = "NULL";
 110   std::string subenums_array_ref = "NULL";
 111   const upb_MiniTable* mt_32 = pools.GetMiniTable32(message);
 112   const upb_MiniTable* mt_64 = pools.GetMiniTable64(message);
 113   std::map<int, std::string> subs;
 114   absl::flat_hash_set<const upb_MiniTable*> seen;
 115
 116   // Construct map of sub messages by field number.
 117   for (int i = 0; i < mt_64->UPB_PRIVATE(field_count); i++) {
 118     const upb_MiniTableField* f = &mt_64->UPB_PRIVATE(fields)[i];
 119     uint32_t index = f->UPB_PRIVATE(submsg_index);
 120     if (index != kUpb_NoSub) {
 121       const int f_number = upb_MiniTableField_Number(f);
 122       upb::FieldDefPtr field = message.FindFieldByNumber(f_number);
 123       auto pair = subs.emplace(index, GetSub(field, false));
 124       ABSL_CHECK(pair.second);
 125       if (options.one_output_per_message && field.IsSubMessage() &&
 126           IsCrossFile(field) && !upb_MiniTableField_IsMap(f)) {
 127         if (seen.insert(pools.GetMiniTable64(field.message_type())).second) {
 128           output(
 129               "__attribute__((weak)) const upb_MiniTable* $0 ="
 130               " &UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken);\n",
 131               MessagePtrVarName(field.message_type()));
 132         }
 133       }
 134     }
 135   }
 136   // Write upb_MiniTableSubInternal table for sub messages referenced from
 137   // fields.
 138   if (!subs.empty()) {
 139     std::string submsgs_array_name =
 140         MiniTableSubMessagesVarName(message.full_name());
 141     submsgs_array_ref = "&" + submsgs_array_name + "[0]";
 142     output("static const upb_MiniTableSubInternal $0[$1] = {\n",
 143            submsgs_array_name, subs.size());
 144
 145     int i = 0;
 146     for (const auto& pair : subs) {
 147       ABSL_CHECK(pair.first == i++);
 148       output("  $0,\n", pair.second);
 149     }
 150
 151     output("};\n\n");
 152   }
 153
 154   // Write upb_MiniTableField table.
 155   if (mt_64->UPB_PRIVATE(field_count) > 0) {
 156     std::string fields_array_name = MiniTableFieldsVarName(message.full_name());
 157     fields_array_ref = "&" + fields_array_name + "[0]";
 158     output("static const upb_MiniTableField $0[$1] = {\n", fields_array_name,
 159            mt_64->UPB_PRIVATE(field_count));
 160     for (int i = 0; i < mt_64->UPB_PRIVATE(field_count); i++) {
 161       WriteMessageField(message.FindFieldByNumber(
 162                             mt_64->UPB_PRIVATE(fields)[i].UPB_PRIVATE(number)),
 163                         &mt_64->UPB_PRIVATE(fields)[i],
 164                         &mt_32->UPB_PRIVATE(fields)[i], output);
 165     }
 166     output("};\n\n");
 167   }
 168
 169   std::vector<TableEntry> table;
 170   uint8_t table_mask = ~0;
 171
 172   table = FastDecodeTable(message, pools);
 173
 174   if (table.size() > 1) {
 175     UPB_ASSERT((table.size() & (table.size() - 1)) == 0);
 176     table_mask = (table.size() - 1) << 3;
 177   }
 178
 179   std::string msgext = "kUpb_ExtMode_NonExtendable";
 180
 181   if (message.extension_range_count()) {
 182     if (UPB_DESC(MessageOptions_message_set_wire_format)(message.options())) {
 183       msgext = "kUpb_ExtMode_IsMessageSet";
 184     } else {
 185       msgext = "kUpb_ExtMode_Extendable";
 186     }
 187   }
 188
 189   output("const upb_MiniTable $0 = {\n", MessageVarName(message));
 190   output("  $0,\n", submsgs_array_ref);
 191   output("  $0,\n", fields_array_ref);
 192   output("  $0, $1, $2, $3, UPB_FASTTABLE_MASK($4), $5,\n",
 193          ArchDependentSize(mt_32->UPB_PRIVATE(size), mt_64->UPB_PRIVATE(size)),
 194          mt_64->UPB_PRIVATE(field_count), msgext,
 195          mt_64->UPB_PRIVATE(dense_below), table_mask,
 196          mt_64->UPB_PRIVATE(required_count));
 197   output("#ifdef UPB_TRACING_ENABLED\n");
 198   output("  \"$0\",\n", message.full_name());
 199   output("#endif\n");
 200   if (!table.empty()) {
 201     output("  UPB_FASTTABLE_INIT({\n");
 202     for (const auto& ent : table) {
 203       output("    {0x$1, &$0},\n", ent.first,
 204              absl::StrCat(absl::Hex(ent.second, absl::kZeroPad16)));
 205     }
 206     output("  })\n");
 207   }
 208   output("};\n\n");
 209   output("const upb_MiniTable* $0 = &$1;\n", MessagePtrVarName(message),
 210          MessageVarName(message));
 211 }
 212
 213 void WriteEnum(upb::EnumDefPtr e, Output& output) {
 214   std::string values_init = "{\n";
 215   const upb_MiniTableEnum* mt = e.mini_table();
 216   uint32_t value_count =
 217       (mt->UPB_PRIVATE(mask_limit) / 32) + mt->UPB_PRIVATE(value_count);
 218   for (uint32_t i = 0; i < value_count; i++) {
 219     absl::StrAppend(&values_init, "                0x",
 220                     absl::Hex(mt->UPB_PRIVATE(data)[i]), ",\n");
 221   }
 222   values_init += "    }";
 223
 224   output(
 225       R"cc(
 226         const upb_MiniTableEnum $0 = {
 227             $1,
 228             $2,
 229             $3,
 230         };
 231       )cc",
 232       EnumVarName(e), mt->UPB_PRIVATE(mask_limit), mt->UPB_PRIVATE(value_count),
 233       values_init);
 234   output("\n");
 235 }
 236
 237 void WriteExtension(const DefPoolPair& pools, upb::FieldDefPtr ext,
 238                     Output& output) {
 239   output("UPB_LINKARR_APPEND(upb_AllExts)\n");
 240   output("const upb_MiniTableExtension $0 = {\n  ", ExtensionVarName(ext));
 241   output("$0,\n", FieldInitializer(pools, ext));
 242   output("  &$0,\n", MessageVarName(ext.containing_type()));
 243   output("  $0,\n", GetSub(ext, true));
 244   output("\n};\n");
 245 }
 246
 247 }  // namespace
 248
 249 void WriteMiniTableHeader(const DefPoolPair& pools, upb::FileDefPtr file,
 250                           const MiniTableOptions& options, Output& output) {
 251   output(FileWarning(file.name()));
 252   output(
 253       "#ifndef $0_UPB_MINITABLE_H_\n"
 254       "#define $0_UPB_MINITABLE_H_\n\n"
 255       "#include \"upb/generated_code_support.h\"\n",
 256       IncludeGuard(file.name()));
 257
 258   for (int i = 0; i < file.public_dependency_count(); i++) {
 259     if (i == 0) {
 260       output("/* Public Imports. */\n");
 261     }
 262     output("#include \"$0\"\n",
 263            HeaderFilename(file.public_dependency(i), options.bootstrap));
 264     if (i == file.public_dependency_count() - 1) {
 265       output("\n");
 266     }
 267   }
 268
 269   output(
 270       "\n"
 271       "// Must be last.\n"
 272       "#include \"upb/port/def.inc\"\n"
 273       "\n"
 274       "#ifdef __cplusplus\n"
 275       "extern \"C\" {\n"
 276       "#endif\n"
 277       "\n");
 278
 279   const std::vector<upb::MessageDefPtr> this_file_messages =
 280       SortedMessages(file);
 281   const std::vector<upb::FieldDefPtr> this_file_exts = SortedExtensions(file);
 282
 283   for (auto message : this_file_messages) {
 284     output("extern const upb_MiniTable $0;\n", MessageVarName(message));
 285     output("extern const upb_MiniTable* $0;\n", MessagePtrVarName(message));
 286   }
 287   for (auto ext : this_file_exts) {
 288     output("extern const upb_MiniTableExtension $0;\n", ExtensionVarName(ext));
 289   }
 290
 291   output("\n");
 292
 293   std::vector<upb::EnumDefPtr> this_file_enums =
 294       SortedEnums(file, kClosedEnums);
 295
 296   for (const auto enumdesc : this_file_enums) {
 297     output("extern const upb_MiniTableEnum $0;\n", EnumVarName(enumdesc));
 298   }
 299
 300   output("extern const upb_MiniTableFile $0;\n\n", FileVarName(file));
 301
 302   output(
 303       "#ifdef __cplusplus\n"
 304       "}  /* extern \"C\" */\n"
 305       "#endif\n"
 306       "\n"
 307       "#include \"upb/port/undef.inc\"\n"
 308       "\n"
 309       "#endif  /* $0_UPB_MINITABLE_H_ */\n",
 310       IncludeGuard(file.name()));
 311 }
 312
 313 void WriteMiniTableSourceIncludes(upb::FileDefPtr file,
 314                                   const MiniTableOptions& options,
 315                                   Output& output) {
 316   output(FileWarning(file.name()));
 317
 318   output(
 319       "#include <stddef.h>\n"
 320       "#include \"upb/generated_code_support.h\"\n"
 321       "#include \"$0\"\n",
 322       HeaderFilename(file, options.bootstrap));
 323
 324   for (int i = 0; i < file.dependency_count(); i++) {
 325     if (options.strip_nonfunctional_codegen &&
 326         google::protobuf::compiler::IsKnownFeatureProto(file.dependency(i).name())) {
 327       // Strip feature imports for editions codegen tests.
 328       continue;
 329     }
 330     output("#include \"$0\"\n",
 331            HeaderFilename(file.dependency(i), options.bootstrap));
 332   }
 333
 334   output(
 335       "\n"
 336       "// Must be last.\n"
 337       "#include \"upb/port/def.inc\"\n"
 338       "\n");
 339
 340   output(
 341       "extern const struct upb_MiniTable "
 342       "UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken);\n");
 343 }
 344
 345 void WriteMiniTableSource(const DefPoolPair& pools, upb::FileDefPtr file,
 346                           const MiniTableOptions& options, Output& output) {
 347   WriteMiniTableSourceIncludes(file, options, output);
 348
 349   std::vector<upb::MessageDefPtr> messages = SortedMessages(file);
 350   std::vector<upb::FieldDefPtr> extensions = SortedExtensions(file);
 351   std::vector<upb::EnumDefPtr> enums = SortedEnums(file, kClosedEnums);
 352
 353   if (options.one_output_per_message) {
 354     for (auto message : messages) {
 355       output("extern const upb_MiniTable* $0;\n", MessagePtrVarName(message));
 356     }
 357     for (const auto e : enums) {
 358       output("extern const upb_MiniTableEnum $0;\n", EnumVarName(e));
 359     }
 360     for (const auto ext : extensions) {
 361       output("extern const upb_MiniTableExtension $0;\n",
 362              ExtensionVarName(ext));
 363     }
 364   } else {
 365     for (auto message : messages) {
 366       WriteMessage(message, pools, options, output);
 367     }
 368     for (const auto e : enums) {
 369       WriteEnum(e, output);
 370     }
 371     for (const auto ext : extensions) {
 372       WriteExtension(pools, ext, output);
 373     }
 374   }
 375
 376   // Messages.
 377   if (!messages.empty()) {
 378     output("static const upb_MiniTable *$0[$1] = {\n", kMessagesInit,
 379            messages.size());
 380     for (auto message : messages) {
 381       output("  &$0,\n", MessageVarName(message));
 382     }
 383     output("};\n");
 384     output("\n");
 385   }
 386
 387   // Enums.
 388   if (!enums.empty()) {
 389     output("static const upb_MiniTableEnum *$0[$1] = {\n", kEnumsInit,
 390            enums.size());
 391     for (const auto e : enums) {
 392       output("  &$0,\n", EnumVarName(e));
 393     }
 394     output("};\n");
 395     output("\n");
 396   }
 397
 398   if (!extensions.empty()) {
 399     // Extensions.
 400     output(
 401         "\n"
 402         "static const upb_MiniTableExtension *$0[$1] = {\n",
 403         kExtensionsInit, extensions.size());
 404
 405     for (auto ext : extensions) {
 406       output("  &$0,\n", ExtensionVarName(ext));
 407     }
 408
 409     output(
 410         "};\n"
 411         "\n");
 412   }
 413
 414   output("const upb_MiniTableFile $0 = {\n", FileVarName(file));
 415   output("  $0,\n", messages.empty() ? "NULL" : kMessagesInit);
 416   output("  $0,\n", enums.empty() ? "NULL" : kEnumsInit);
 417   output("  $0,\n", extensions.empty() ? "NULL" : kExtensionsInit);
 418   output("  $0,\n", messages.size());
 419   output("  $0,\n", enums.size());
 420   output("  $0,\n", extensions.size());
 421   output("};\n\n");
 422
 423   output("#include \"upb/port/undef.inc\"\n");
 424   output("\n");
 425 }
 426
 427 std::string MultipleSourceFilename(upb::FileDefPtr file,
 428                                    absl::string_view full_name, int* i) {
 429   *i += 1;
 430   return absl::StrCat(StripExtension(file.name()), ".upb_weak_minitables/",
 431                       *i, ".upb.c");
 432 }
 433
 434 void WriteMiniTableMultipleSources(const DefPoolPair& pools,
 435                                    upb::FileDefPtr file,
 436                                    const MiniTableOptions& options,
 437                                    Plugin* plugin) {
 438   std::vector<upb::MessageDefPtr> messages = SortedMessages(file);
 439   std::vector<upb::FieldDefPtr> extensions = SortedExtensions(file);
 440   std::vector<upb::EnumDefPtr> enums = SortedEnums(file, kClosedEnums);
 441   int i = 0;
 442
 443   for (auto message : messages) {
 444     Output output;
 445     WriteMiniTableSourceIncludes(file, options, output);
 446     WriteMessage(message, pools, options, output);
 447     plugin->AddOutputFile(MultipleSourceFilename(file, message.full_name(), &i),
 448                           output.output());
 449   }
 450   for (const auto e : enums) {
 451     Output output;
 452     WriteMiniTableSourceIncludes(file, options, output);
 453     WriteEnum(e, output);
 454     plugin->AddOutputFile(MultipleSourceFilename(file, e.full_name(), &i),
 455                           output.output());
 456   }
 457   for (const auto ext : extensions) {
 458     Output output;
 459     WriteMiniTableSourceIncludes(file, options, output);
 460     WriteExtension(pools, ext, output);
 461     plugin->AddOutputFile(MultipleSourceFilename(file, ext.full_name(), &i),
 462                           output.output());
 463   }
 464 }
 465
 466 }  // namespace generator
 467 }  // namespace upb