upb_generator/bootstrap_compiler.bzl

   1 """Macros that implement bootstrapping for the upb code generator."""
   2
   3 load(
   4     "//bazel:upb_minitable_proto_library.bzl",
   5     "upb_minitable_proto_library",
   6 )
   7 load(
   8     "//bazel:upb_proto_library.bzl",
   9     "upb_proto_library",
  10 )
  11 load(
  12     "//upb/cmake:build_defs.bzl",
  13     "staleness_test",
  14 )
  15
  16 _stages = ["_stage0", "_stage1", ""]
  17 _protoc = "//:protoc"
  18
  19 _is_google3 = False
  20 _extra_proto_path = "-I$$(dirname $(location @com_google_protobuf//:descriptor_proto_srcs))/../.. "
  21
  22 # This visibility is used automatically for anything used by the bootstrapping process.
  23 _bootstrap_visibility = [
  24     "//upb_generator:__subpackages__",
  25     "//upb/reflection:__pkg__",
  26     "//upb:__pkg__",  # For the amalgamations.
  27     "//python/dist:__pkg__",  # For the Python source package.
  28 ]
  29
  30 def _stage_visibility(stage, visibility):
  31     return visibility if stage == "" else _bootstrap_visibility
  32
  33 def _upbc(generator, stage):
  34     if generator == "upb":
  35         return "//upb_generator/c:protoc-gen-upb" + _stages[stage]
  36     else:
  37         return "//upb_generator/minitable:protoc-gen-upb_minitable" + _stages[stage]
  38
  39 def bootstrap_cc_library(name, visibility = [], deps = [], bootstrap_deps = [], **kwargs):
  40     """A version of cc_library() that is augmented to allow for bootstrapping the compiler.
  41
  42     In addition to the normal cc_library() target, this rule will also generate _stage0 and _stage1
  43     targets that are used internally for bootstrapping, and will automatically have bootstrap
  44     visibility. However the final target will use the normal visibility, and will behave like a
  45     normal cc_library() target.
  46
  47     Args:
  48         name: Name of this rule.  This name will resolve to a upb_proto_library().
  49         deps: Normal cc_library() deps.
  50         bootstrap_deps: Special bootstrap_upb_proto_library() or bootstrap_cc_library() deps.
  51         visibility: Visibility of the final target.
  52         **kwargs: Other arguments that will be passed through to cc_library().
  53           upb_proto_library().
  54     """
  55     for stage in _stages:
  56         native.cc_library(
  57             name = name + stage,
  58             deps = deps + [dep + stage for dep in bootstrap_deps],
  59             visibility = _stage_visibility(stage, visibility),
  60             **kwargs
  61         )
  62
  63 def bootstrap_cc_binary(name, visibility = [], deps = [], bootstrap_deps = [], **kwargs):
  64     """A version of cc_binary() that is augmented to allow for bootstrapping the compiler.
  65
  66     In addition to the normal cc_binary() target, this rule will also generate _stage0 and _stage1
  67     targets that are used internally for bootstrapping, and will automatically have bootstrap
  68     visibility. However the final target will use the normal visibility, and will behave like a
  69     normal cc_binary() target.
  70
  71     Args:
  72         name: Name of this rule.  This name will resolve to a upb_proto_library().
  73         deps: Normal cc_library() deps.
  74         bootstrap_deps: Special bootstrap_upb_proto_library() or bootstrap_cc_library() deps.
  75         visibility: Visibility of the final target.
  76         **kwargs: Other arguments that will be passed through to cc_binary().
  77           upb_proto_library().
  78     """
  79     for stage in _stages:
  80         native.cc_binary(
  81             name = name + stage,
  82             deps = deps + [dep + stage for dep in bootstrap_deps],
  83             visibility = _stage_visibility(stage, visibility),
  84             **kwargs
  85         )
  86
  87 def _generated_file(proto, stage, generator, suffix):
  88     stripped = proto[:-len(".proto")]
  89     return "{}/{}.{}.{}".format(stage, stripped, generator, suffix)
  90
  91 def _generated_files(protos, stage, generator, suffix):
  92     return [_generated_file(proto, stage, generator, suffix) for proto in protos]
  93
  94 def _generated_hdrs_and_srcs(protos, stage, generator):
  95     ret = _generated_files(protos, stage, generator, "h")
  96     if generator != "upb" or stage == "stage0":
  97         ret += _generated_files(protos, stage, generator, "c")
  98     return ret
  99
 100 def _stage0_proto_staleness_test(name, src_files, src_rules, strip_prefix):
 101     native.genrule(
 102         name = name + "_generate_bootstrap",
 103         srcs = src_rules,
 104         outs = ["bootstrap_generated_sources/" + f for f in _generated_hdrs_and_srcs(src_files, "stage0", "upb")],
 105         tools = [_protoc, _upbc("upb", 0)],
 106         cmd =
 107             "$(location " + _protoc + ") " +
 108             "-I$(GENDIR)/" + strip_prefix + " " + _extra_proto_path +
 109             "--plugin=protoc-gen-upb=$(location " + _upbc("upb", 0) + ") " +
 110             "--upb_out=bootstrap_stage=0:$(@D)/bootstrap_generated_sources/stage0 " +
 111             " ".join(src_files),
 112     )
 113
 114     staleness_test(
 115         name = name + "_stage0_staleness_test",
 116         outs = _generated_hdrs_and_srcs(src_files, "stage0", "upb"),
 117         generated_pattern = "bootstrap_generated_sources/%s",
 118         target_files = native.glob(["stage0/**"]),
 119         # To avoid skew problems for descriptor.proto/pluging.proto between
 120         # GitHub repos.  It's not critical that the checked-in protos are up to
 121         # date for every change, they just needs to be complete enough to have
 122         # everything needed by the code generator itself.
 123         tags = ["manual"],
 124     )
 125
 126 def _generate_stage1_proto(name, src_files, src_rules, generator, kwargs):
 127     native.genrule(
 128         name = "gen_{}_{}_stage1".format(name, generator),
 129         srcs = src_rules,
 130         outs = _generated_hdrs_and_srcs(src_files, "stage1", generator),
 131         cmd = "$(location " + _protoc + ") " +
 132               "--plugin=protoc-gen-" + generator +
 133               "=$(location " + _upbc(generator, 0) + ") " + _extra_proto_path +
 134               "--" + generator + "_out=bootstrap_stage=1:$(RULEDIR)/stage1 " +
 135               " ".join(src_files),
 136         visibility = _bootstrap_visibility,
 137         tools = [
 138             _protoc,
 139             _upbc(generator, 0),
 140         ],
 141         **kwargs
 142     )
 143
 144 def _cmake_staleness_test(name, src_files, proto_lib_deps, **kwargs):
 145     upb_minitable_proto_library(
 146         name = name + "_minitable",
 147         deps = proto_lib_deps,
 148         **kwargs
 149     )
 150
 151     # Copy the final gencode for staleness comparison
 152     files = _generated_hdrs_and_srcs(src_files, "cmake", "upb") + \
 153             _generated_hdrs_and_srcs(src_files, "cmake", "upb_minitable")
 154     genrule = 0
 155     for src in files:
 156         genrule += 1
 157         native.genrule(
 158             name = name + "_copy_gencode_%d" % genrule,
 159             outs = ["generated_sources/" + src],
 160             srcs = [name + "_upb_proto", name + "_minitable"],
 161             cmd = """
 162                 mkdir -p $(@D)
 163                 for src in $(SRCS); do
 164                     if [[ $$src == *%s ]]; then
 165                         cp -f $$src $(@D) || echo 'copy failed!'
 166                     fi
 167                 done
 168             """ % src[src.rfind("/"):],
 169         )
 170
 171     # Keep bazel gencode in sync with our checked-in sources needed for cmake builds.
 172     staleness_test(
 173         name = name + "_staleness_test",
 174         outs = files,
 175         generated_pattern = "generated_sources/%s",
 176         tags = ["manual"],
 177     )
 178
 179 def bootstrap_upb_proto_library(
 180         name,
 181         bootstrap_hdr,
 182         google3_src_files,
 183         google3_src_rules,
 184         oss_src_files,
 185         oss_src_rules,
 186         oss_strip_prefix,
 187         proto_lib_deps,
 188         deps = [],
 189         **kwargs):
 190     """A version of upb_proto_library() that is augmented to allow for bootstrapping the compiler.
 191
 192     Note that this rule is only intended to be used by bootstrap_cc_library() targets. End users
 193     should use the normal upb_proto_library() targets. As a result, we don't have a visibility
 194     parameter: all targets will automatically have bootstrap visibility.
 195
 196     Args:
 197         name: Name of this rule.  This name will resolve to a upb_proto_library().
 198         bootstrap_hdr: The forwarding header that exposes the generated code, taking into account
 199           the current stage.
 200         google3_src_files: Google3 filenames of .proto files that should be built by this rule.
 201           The names should be relative to the depot base.
 202         google3_src_rules: Target names of the Blaze rules that will provide these filenames.
 203         oss_src_files: OSS filenames of .proto files that should be built by this rule.
 204         oss_src_rules: Target names of the Bazel rules that will provide these filenames.
 205         oss_strip_prefix: Prefix that should be stripped from OSS file names.
 206         proto_lib_deps: proto_library() rules that we will use to build the protos when we are
 207           not bootstrapping.
 208         deps: other bootstrap_upb_proto_library() rules that this one depends on.
 209         **kwargs: Other arguments that will be passed through to cc_library(), genrule(), and
 210           upb_proto_library().
 211     """
 212     _stage0_proto_staleness_test(name, oss_src_files, oss_src_rules, oss_strip_prefix)
 213
 214     # stage0 uses checked-in protos, and has no MiniTable.
 215     native.cc_library(
 216         name = name + "_stage0",
 217         srcs = _generated_hdrs_and_srcs(oss_src_files, "stage0", "upb"),
 218         hdrs = [bootstrap_hdr],
 219         visibility = _bootstrap_visibility,
 220         defines = ["UPB_BOOTSTRAP_STAGE=0"],
 221         deps = [
 222             "//upb:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
 223             "//upb:mini_table",
 224         ] + [dep + "_stage0" for dep in deps],
 225         **kwargs
 226     )
 227
 228     src_files = google3_src_files if _is_google3 else oss_src_files
 229     src_rules = google3_src_rules if _is_google3 else oss_src_rules
 230
 231     # Generate stage1 protos (C API and MiniTables) using stage0 compiler.
 232     _generate_stage1_proto(name, src_files, src_rules, "upb", kwargs)
 233     _generate_stage1_proto(name, src_files, src_rules, "upb_minitable", kwargs)
 234
 235     native.cc_library(
 236         name = name + "_minitable_stage1",
 237         srcs = _generated_files(src_files, "stage1", "upb_minitable", "c"),
 238         hdrs = _generated_files(src_files, "stage1", "upb_minitable", "h"),
 239         visibility = _bootstrap_visibility,
 240         defines = ["UPB_BOOTSTRAP_STAGE=1"],
 241         deps = [
 242             "//upb:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
 243         ] + [dep + "_minitable_stage1" for dep in deps],
 244         **kwargs
 245     )
 246     native.cc_library(
 247         name = name + "_stage1",
 248         srcs = _generated_files(src_files, "stage1", "upb", "h"),
 249         hdrs = [bootstrap_hdr],
 250         visibility = _bootstrap_visibility,
 251         defines = ["UPB_BOOTSTRAP_STAGE=1"],
 252         deps = [
 253             "//upb:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
 254             ":" + name + "_minitable_stage1",
 255         ] + [dep + "_minitable_stage1" for dep in deps],
 256         **kwargs
 257     )
 258
 259     # The final protos are generated via normal upb_proto_library().
 260     upb_proto_library(
 261         name = name + "_upb_proto",
 262         deps = proto_lib_deps,
 263         **kwargs
 264     )
 265     native.cc_library(
 266         name = name,
 267         hdrs = [bootstrap_hdr],
 268         deps = [name + "_upb_proto"],
 269         visibility = _bootstrap_visibility,
 270         **kwargs
 271     )
 272
 273     _cmake_staleness_test(name, src_files, proto_lib_deps, **kwargs)