diff --git a/kotlin/defs.bzl b/kotlin/defs.bzl index d7fb2bf..af65756 100644 --- a/kotlin/defs.bzl +++ b/kotlin/defs.bzl @@ -19,13 +19,17 @@ load( _kt_jvm_binary = "kt_jvm_binary", _kt_jvm_test = "kt_jvm_test", ) +load( + "//kotlin/internal:grpc_library.bzl", + _kt_jvm_grpc_library = "kt_jvm_grpc_library", + _kt_jvm_grpc_proto_library = "kt_jvm_grpc_proto_library", +) load( "//kotlin/internal:library.bzl", _kt_jvm_library = "kt_jvm_library", ) load( "//kotlin/internal:proto_library.bzl", - _kt_jvm_grpc_proto_library = "kt_jvm_grpc_proto_library", _kt_jvm_proto_library = "kt_jvm_proto_library", ) @@ -33,4 +37,5 @@ kt_jvm_library = _kt_jvm_library kt_jvm_test = _kt_jvm_test kt_jvm_binary = _kt_jvm_binary kt_jvm_proto_library = _kt_jvm_proto_library +kt_jvm_grpc_library = _kt_jvm_grpc_library kt_jvm_grpc_proto_library = _kt_jvm_grpc_proto_library diff --git a/kotlin/internal/common.bzl b/kotlin/internal/common.bzl new file mode 100644 index 0000000..a848546 --- /dev/null +++ b/kotlin/internal/common.bzl @@ -0,0 +1,99 @@ +# Copyright 2023 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common defs.""" + +def get_real_short_path(file): + """Returns the correct short path for a `File`. + + Args: + file: the `File` to return the short path for. + + Returns: + The short path for `file`, handling any non-standard path segments if + it's from external repositories. + """ + + # For some reason, files from other archives have short paths that look like: + # ../com_google_protobuf/google/protobuf/descriptor.proto + short_path = file.short_path + if short_path.startswith("../"): + second_slash = short_path.index("/", 3) + short_path = short_path[second_slash + 1:] + + # Sometimes it has another few prefixes like: + # _virtual_imports/any_proto/google/protobuf/any.proto + # benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto + # We want just google/protobuf/any.proto. + virtual_imports = "_virtual_imports/" + if virtual_imports in short_path: + short_path = short_path.split(virtual_imports)[1].split("/", 1)[1] + return short_path + +def create_srcjar(ctx, zipper, output_jar, *args): + """Bundles source files into a srcjar. + + Args: + ctx: rule context + zipper: zipper executable + output_jar: output srcjar + *args: input directories + """ + input_dirs = args + + zipper_args = ctx.actions.args() + zipper_args.add("c", output_jar) + zipper_args.add_all(input_dirs) + + ctx.actions.run( + outputs = [output_jar], + inputs = input_dirs, + executable = zipper, + arguments = [zipper_args], + mnemonic = "SrcJar", + progress_message = "Generating srcjar for " + ctx.label.name, + toolchain = None, + ) + +def merge_srcjars(ctx, zipper, extract_srcjars, output_jar, *args): + """Merges multiple srcjars into a single srcjar. + + Args: + ctx: rule context + zipper: zipper executable + extract_srcjars: extract_srcjars executable + output_jar: output srcjar + *args: input srcjars + """ + input_jars = args + + tmp_dir_name = ctx.label.name + "_srcjars" + tmp_dir = ctx.actions.declare_directory(tmp_dir_name) + + merge_args = ctx.actions.args() + merge_args.add(zipper) + merge_args.add(tmp_dir.path) + merge_args.add_all(input_jars) + + ctx.actions.run( + outputs = [tmp_dir], + inputs = input_jars, + executable = extract_srcjars, + tools = [zipper], + arguments = [merge_args], + mnemonic = "ExtractSrcJars", + toolchain = None, + ) + + create_srcjar(ctx, zipper, output_jar, tmp_dir) diff --git a/kotlin/internal/grpc_library.bzl b/kotlin/internal/grpc_library.bzl new file mode 100644 index 0000000..bc776c3 --- /dev/null +++ b/kotlin/internal/grpc_library.bzl @@ -0,0 +1,255 @@ +# Copyright 2023 The Cross-Media Measurement Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rules/macros for gRPC libraries.""" + +load("@rules_java//java:defs.bzl", "java_library") +load("//kotlin/internal:common.bzl", "create_srcjar", "get_real_short_path") +load("//kotlin/internal:library.bzl", "kt_jvm_library") +load("//kotlin/internal:proto_library.bzl", "kt_jvm_proto_library") + +_KT_JVM_GRPC_DEPS = [ + Label("//imports/io/gprc:api"), + Label("//imports/io/gprc/protobuf"), + Label("//imports/io/gprc/stub"), + Label("//imports/io/gprc/kotlin:stub"), +] + +def _generate_java_grpc_extensions( + ctx, + protoc, + grpc_java_plugin, + proto_lib, + output_dir): + """Generates code for Kotlin proto extensions.""" + proto_info = proto_lib[ProtoInfo] + srcs = proto_info.direct_sources + transitive_descriptor_set = proto_info.transitive_descriptor_sets + + args = ctx.actions.args() + args.set_param_file_format("multiline") + args.use_param_file("@%s") + args.add_joined( + transitive_descriptor_set, + join_with = ctx.configuration.host_path_separator, + format_joined = "--descriptor_set_in=%s", + ) + args.add_all(srcs, map_each = get_real_short_path) + args.add( + grpc_java_plugin, + format = "--plugin=protoc-gen-grpc-java=%s", + ) + args.add("--grpc-java_out=" + output_dir.path) + + ctx.actions.run( + inputs = depset( + srcs, + transitive = [transitive_descriptor_set], + ), + outputs = [output_dir], + executable = protoc, + arguments = [args], + tools = [grpc_java_plugin], + mnemonic = "JavaGrpcProtoc", + progress_message = "Generating Java gRPC extensions for " + ctx.label.name, + toolchain = None, + ) + +def _generate_kotlin_grpc_extensions( + ctx, + grpc_kt_generator, + proto_lib, + output_dir): + """Generates code for Kotlin gRPC extensions.""" + proto_info = proto_lib[ProtoInfo] + direct_descriptor_set = proto_info.direct_descriptor_set + transitive_descriptor_set = proto_info.transitive_descriptor_sets + + generator_args = ctx.actions.args() + generator_args.add(output_dir.path) + generator_args.add_all([direct_descriptor_set]) + generator_args.add("--") + generator_args.add_all(transitive_descriptor_set) + + ctx.actions.run( + inputs = depset( + [direct_descriptor_set], + transitive = [transitive_descriptor_set], + ), + outputs = [output_dir], + arguments = [generator_args], + executable = grpc_kt_generator, + mnemonic = "KtGrpcGenerator", + progress_message = "Generating Kotlin gRPC extensions for " + ctx.label.name, + toolchain = None, + ) + +def _kt_jvm_grpc_library_helper_impl(ctx): + name = ctx.attr.name + proto_dep = ctx.attr.proto_dep + + # TODO(@SanjayVas): Compile generated Java code, collecting JavaInfo. + java_grpc_src_dir_name = name + "_java_grpc_srcs" + java_grpc_src_dir = ctx.actions.declare_directory(java_grpc_src_dir_name) + _generate_java_grpc_extensions( + ctx, + ctx.executable._protoc, + ctx.executable._grpc_java_plugin, + proto_dep, + java_grpc_src_dir, + ) + + kt_grpc_src_dir_name = name + "_kt_jvm_grpc_srcs" + kt_grpc_src_dir = ctx.actions.declare_directory(kt_grpc_src_dir_name) + _generate_kotlin_grpc_extensions( + ctx, + ctx.executable._grpc_kt_generator, + proto_dep, + kt_grpc_src_dir, + ) + + create_srcjar( + ctx, + ctx.executable._zipper, + ctx.outputs.srcjar, + java_grpc_src_dir, + kt_grpc_src_dir, + ) + +_kt_jvm_grpc_library_helper = rule( + implementation = _kt_jvm_grpc_library_helper_impl, + attrs = { + "proto_dep": attr.label( + providers = [ProtoInfo], + mandatory = True, + ), + "srcjar": attr.output( + doc = "Generated Java source jar.", + mandatory = True, + ), + "_zipper": attr.label( + default = Label("@bazel_tools//tools/zip:zipper"), + cfg = "exec", + executable = True, + ), + "_protoc": attr.label( + default = Label("@com_google_protobuf//:protoc"), + cfg = "exec", + executable = True, + ), + "_grpc_kt_generator": attr.label( + default = Label(":KotlinGrpcGenerator"), + cfg = "exec", + executable = True, + ), + "_grpc_java_plugin": attr.label( + default = Label(":protoc_gen_grpc_java"), + cfg = "exec", + executable = True, + ), + }, +) + +def kt_jvm_grpc_library( + name, + srcs = None, + deps = None, + visibility = None, + **kwargs): + """Generates and compiles gRPC extensions from a proto_library. + + Args: + name: a name for the target + srcs: exactly one proto_library target + deps: exactly one JVM proto_library target for srcs, e.g. a kt_jvm_proto_library + visibility: standard visibility attribute + **kwargs: additional arguments for resulting target + """ + srcs = srcs or [] + deps = deps or [] + if len(srcs) != 1: + fail("Expected exactly one src", "srcs") + if len(deps) != 1: + fail("Expected exactly one dep", "deps") + + helper_name = name + "_DO_NOT_DEPEND_kt_jvm_grpc_gen" + srcjar_name = helper_name + ".srcjar" + _kt_jvm_grpc_library_helper( + name = helper_name, + proto_dep = srcs[0], + srcjar = srcjar_name, + visibility = ["//visibility:private"], + **kwargs + ) + + kt_jvm_library( + name = name, + srcs = [":" + srcjar_name], + deps = deps + _KT_JVM_GRPC_DEPS, + exports = _KT_JVM_GRPC_DEPS, + visibility = visibility, + kotlinc_opts = Label(":proto_gen_kt_options"), + **kwargs + ) + +_PROTO_LIBRARY_SUFFIX = "_proto" + +def kt_jvm_grpc_proto_library(name, deps = None, **kwargs): + """Wrapper for generating Kotlin JVM protobuf APIs and gRPC extensions. + + Given a proto_library named `_proto`, this will create additional + `_kt_jvm_proto` and `_kt_jvm_grpc` targets. + + Args: + name: a name for the target + deps: exactly one proto_library target + **kwargs: other args to pass to the resulting target + """ + deps = deps or [] + if len(deps) != 1: + fail("Expected exactly one dep", "deps") + + proto_label = native.package_relative_label(deps[0]) + proto_name = proto_label.name + if not proto_name.endswith(_PROTO_LIBRARY_SUFFIX): + fail("proto_library target names should end with '{suffix}'".format( + suffix = _PROTO_LIBRARY_SUFFIX, + )) + name_prefix = proto_label.name.removesuffix(_PROTO_LIBRARY_SUFFIX) + + kt_jvm_proto_name = name_prefix + "_kt_jvm_proto" + kt_jvm_proto_label = ":" + kt_jvm_proto_name + kt_jvm_proto_library( + name = kt_jvm_proto_name, + deps = deps, + **kwargs + ) + + kt_jvm_grpc_name = name_prefix + "_kt_jvm_grpc" + kt_jvm_grpc_label = ":" + kt_jvm_grpc_name + kt_jvm_grpc_library( + name = kt_jvm_grpc_name, + srcs = deps, + deps = [kt_jvm_proto_label], + **kwargs + ) + + java_library( + name = name, + exports = [ + kt_jvm_proto_label, + kt_jvm_grpc_label, + ], + **kwargs + ) diff --git a/kotlin/internal/proto_library.bzl b/kotlin/internal/proto_library.bzl index 620b076..26abe08 100644 --- a/kotlin/internal/proto_library.bzl +++ b/kotlin/internal/proto_library.bzl @@ -18,6 +18,12 @@ Provides kt_jvm_proto_library to generate Kotlin protos. load("@rules_java//java:defs.bzl", "java_proto_library") load("@rules_proto//proto:defs.bzl", "ProtoInfo") +load( + "//kotlin/internal:common.bzl", + "create_srcjar", + "get_real_short_path", + "merge_srcjars", +) load("//kotlin/internal:library.bzl", "kt_jvm_library") KtProtoLibInfo = provider( @@ -27,34 +33,7 @@ KtProtoLibInfo = provider( }, ) -def _get_real_short_path(file): - """Returns the correct short path for a `File`. - - Args: - file: the `File` to return the short path for. - - Returns: - The short path for `file`, handling any non-standard path segments if - it's from external repositories. - """ - - # For some reason, files from other archives have short paths that look like: - # ../com_google_protobuf/google/protobuf/descriptor.proto - short_path = file.short_path - if short_path.startswith("../"): - second_slash = short_path.index("/", 3) - short_path = short_path[second_slash + 1:] - - # Sometimes it has another few prefixes like: - # _virtual_imports/any_proto/google/protobuf/any.proto - # benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto - # We want just google/protobuf/any.proto. - virtual_imports = "_virtual_imports/" - if virtual_imports in short_path: - short_path = short_path.split(virtual_imports)[1].split("/", 1)[1] - return short_path - -def _generate_proto_extensions(ctx, proto_lib, output_dir): +def _generate_kotlin_proto_extensions(ctx, protoc, proto_lib, output_dir): """Generates code for Kotlin proto extensions.""" proto_info = proto_lib[ProtoInfo] srcs = proto_info.direct_sources @@ -69,110 +48,33 @@ def _generate_proto_extensions(ctx, proto_lib, output_dir): join_with = ctx.configuration.host_path_separator, format_joined = "--descriptor_set_in=%s", ) - args.add_all(srcs, map_each = _get_real_short_path) - if ctx.attr.has_services: - # TODO(@SanjayVas): Compile generated Java code, collecting JavaInfo. - args.add( - ctx.executable._grpc_java_plugin, - format = "--plugin=protoc-gen-grpc-java=%s", - ) - args.add("--grpc-java_out=" + output_dir.path) + args.add_all(srcs, map_each = get_real_short_path) ctx.actions.run( - inputs = depset( - srcs + [ctx.executable._grpc_java_plugin], - transitive = [transitive_descriptor_set], - ), + inputs = depset(srcs, transitive = [transitive_descriptor_set]), outputs = [output_dir], - executable = ctx.executable._protoc, + executable = protoc, arguments = [args], mnemonic = "KtProtoc", progress_message = "Generating Kotlin protobuf extensions for " + ctx.label.name, toolchain = None, ) -def _generate_grpc_extensions(ctx, proto_lib, output_dir): - """Generates code for Kotlin gRPC extensions.""" - proto_info = proto_lib[ProtoInfo] - direct_descriptor_set = depset([proto_info.direct_descriptor_set]) - transitive_descriptor_set = depset(transitive = [proto_info.transitive_descriptor_sets]) - - generator_args = ctx.actions.args() - generator_args.add(output_dir.path) - generator_args.add_all(direct_descriptor_set) - generator_args.add("--") - generator_args.add_all(transitive_descriptor_set) - - ctx.actions.run( - inputs = depset(transitive = [direct_descriptor_set, transitive_descriptor_set]), - outputs = [output_dir], - arguments = [generator_args], - executable = ctx.executable._grpc_generator, - mnemonic = "KtGrpcGenerator", - progress_message = "Generating Kotlin gRPC extensions for " + ctx.label.name, - toolchain = None, - ) - -def _create_srcjar(ctx, input_dir, output_jar): - """Bundles .kt files into a srcjar.""" - zipper_args = ctx.actions.args() - zipper_args.add("c", output_jar) - zipper_args.add_all([input_dir]) - - ctx.actions.run( - outputs = [output_jar], - inputs = [input_dir], - executable = ctx.executable._zipper, - arguments = [zipper_args], - mnemonic = "KtProtoSrcJar", - progress_message = "Generating Kotlin proto srcjar for " + ctx.label.name, - toolchain = None, - ) - -def _merge_srcjars(ctx, input_jars, output_jar): - """Merges multiple srcjars into a single srcjar.""" - tmp_dir_name = ctx.label.name + "_srcjars" - tmp_dir = ctx.actions.declare_directory(tmp_dir_name) - - merge_args = ctx.actions.args() - merge_args.add(ctx.executable._zipper) - merge_args.add(tmp_dir.path) - merge_args.add_all(input_jars) - - ctx.actions.run( - outputs = [tmp_dir], - inputs = input_jars, - executable = ctx.executable._extract_srcjars, - tools = [ctx.executable._zipper], - arguments = [merge_args], - mnemonic = "KtProtoExtractSrcJars", - toolchain = None, - ) - - _create_srcjar(ctx, tmp_dir, output_jar) - def _kt_jvm_proto_aspect_impl(target, ctx): name = target.label.name - srcjars = [] gen_src_dir_name = name + "_kt_jvm_srcs" gen_src_dir = ctx.actions.declare_directory(gen_src_dir_name) - _generate_proto_extensions(ctx, target, gen_src_dir) + _generate_kotlin_proto_extensions( + ctx, + ctx.executable._protoc, + target, + gen_src_dir, + ) srcjar_name = name + "_kt_jvm.srcjar" srcjar = ctx.actions.declare_file(srcjar_name) - _create_srcjar(ctx, gen_src_dir, srcjar) - srcjars.append(srcjar) - - if ctx.attr.has_services: - gen_grpc_src_dir_name = name + "_kt_jvm_grpc_srcs" - gen_grpc_src_dir = ctx.actions.declare_directory(gen_grpc_src_dir_name) - _generate_grpc_extensions(ctx, target, gen_grpc_src_dir) - - grpc_srcjar_name = "_kt_jvm_grpc.srcjar" - grpc_srcjar = ctx.actions.declare_file(grpc_srcjar_name) - _create_srcjar(ctx, gen_grpc_src_dir, grpc_srcjar) - srcjars.append(grpc_srcjar) + create_srcjar(ctx, ctx.executable._zipper, srcjar, gen_src_dir) transitive = [ dep[KtProtoLibInfo].srcjars @@ -180,14 +82,11 @@ def _kt_jvm_proto_aspect_impl(target, ctx): if KtProtoLibInfo in dep ] return [KtProtoLibInfo( - srcjars = depset(direct = srcjars, transitive = transitive), + srcjars = depset(direct = [srcjar], transitive = transitive), )] _kt_jvm_proto_aspect = aspect( attrs = { - "has_services": attr.bool( - default = False, - ), "_zipper": attr.label( default = Label("@bazel_tools//tools/zip:zipper"), cfg = "exec", @@ -198,16 +97,6 @@ _kt_jvm_proto_aspect = aspect( cfg = "exec", executable = True, ), - "_grpc_generator": attr.label( - default = Label(":KotlinGrpcGenerator"), - cfg = "exec", - executable = True, - ), - "_grpc_java_plugin": attr.label( - default = Label(":protoc_gen_grpc_java"), - cfg = "exec", - executable = True, - ), }, implementation = _kt_jvm_proto_aspect_impl, attr_aspects = ["deps"], @@ -216,7 +105,13 @@ _kt_jvm_proto_aspect = aspect( def _kt_jvm_proto_library_helper_impl(ctx): """Implementation of _kt_jvm_proto_library_helper rule.""" proto_lib_info = ctx.attr.proto_dep[KtProtoLibInfo] - _merge_srcjars(ctx, proto_lib_info.srcjars, ctx.outputs.srcjar) + merge_srcjars( + ctx, + ctx.executable._zipper, + ctx.executable._extract_srcjars, + ctx.outputs.srcjar, + *proto_lib_info.srcjars.to_list() + ) _kt_jvm_proto_library_helper = rule( doc = """ @@ -236,9 +131,6 @@ _kt_jvm_proto_library_helper = rule( doc = "Generated Java source jar.", mandatory = True, ), - "has_services": attr.bool( - default = False, - ), "_zipper": attr.label( default = Label("@bazel_tools//tools/zip:zipper"), cfg = "exec", @@ -256,17 +148,10 @@ _kt_jvm_proto_library_helper = rule( _KT_JVM_PROTO_DEPS = [ Label("//imports/com/google/protobuf/kotlin"), ] -_KT_JVM_GRPC_DEPS = [ - Label("//imports/io/gprc:api"), - Label("//imports/io/gprc/protobuf"), - Label("//imports/io/gprc/stub"), - Label("//imports/io/gprc/kotlin:stub"), -] def kt_jvm_proto_library( name, deps = None, - has_services = False, **kwargs): """Generates and compiles Java and Kotlin APIs for a proto_library. @@ -276,7 +161,6 @@ def kt_jvm_proto_library( Args: name: A name for the target deps: Exactly one proto_library target to generate Kotlin APIs for - has_services: Whether to generate code for gRPC services. **kwargs: other args to pass to the ultimate kt_jvm_library target """ deps = deps or [] @@ -284,8 +168,6 @@ def kt_jvm_proto_library( if len(deps) != 1: fail("Expected exactly one dep", "deps") - kt_jvm_deps = [] + _KT_JVM_PROTO_DEPS - java_name = name + "_DO_NOT_DEPEND_java_proto" java_label = ":" + java_name java_proto_library( @@ -293,10 +175,7 @@ def kt_jvm_proto_library( deps = deps, visibility = ["//visibility:private"], ) - kt_jvm_deps.append(java_label) - - if has_services: - kt_jvm_deps.extend(_KT_JVM_GRPC_DEPS) + kt_jvm_deps = _KT_JVM_PROTO_DEPS + [java_label] generated_kt_name = name + "_DO_NOT_DEPEND_generated_kt" generated_srcjar = generated_kt_name + ".srcjar" @@ -304,7 +183,6 @@ def kt_jvm_proto_library( name = generated_kt_name, proto_dep = deps[0], srcjar = generated_srcjar, - has_services = has_services, visibility = ["//visibility:private"], ) @@ -316,21 +194,3 @@ def kt_jvm_proto_library( kotlinc_opts = Label(":proto_gen_kt_options"), **kwargs ) - -def kt_jvm_grpc_proto_library(name, deps = None, **kwargs): - """Generates and compiles Java and Kotlin APIs for a gRPC proto_library. - - For standard attributes, see: - https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes - - Args: - name: A name for the target - deps: Exactly one proto_library target to generate Kotlin APIs for - **kwargs: other args to pass to the ultimate kt_jvm_library target - """ - kt_jvm_proto_library( - name = name, - deps = deps, - has_services = True, - **kwargs - )